OpenCL based ACC-backend and SMM library

.ci/daint.cscs.ch/cray.build.sh

@@ -30,7 +30,7 @@ cd "${SCRATCH}/${BUILD_TAG}.cray"
 
 cmake \
     -DCMAKE_SYSTEM_NAME=CrayLinuxEnvironment \
-    -DUSE_CUDA=ON \
+    -DUSE_ACCEL=cuda \
     -DWITH_GPU=P100 \
     -DBLAS_FOUND=ON -DBLAS_LIBRARIES="-lsci_cray_mpi_mp" \
     -DLAPACK_FOUND=ON -DLAPACK_LIBRARIES="-lsci_cray_mpi_mp" \

.ci/daint.cscs.ch/gnu.build.sh

@@ -28,7 +28,7 @@ cd "${SCRATCH}/${BUILD_TAG}.gnu"
 cmake \
     -DCMAKE_SYSTEM_NAME=CrayLinuxEnvironment \
     -DCMAKE_CROSSCOMPILING_EMULATOR="" \
-    -DUSE_CUDA=ON \
+    -DUSE_ACCEL=cuda \
     -DWITH_GPU=P100 \
     -DBLAS_FOUND=ON -DBLAS_LIBRARIES="-lsci_gnu_mpi_mp" \
     -DLAPACK_FOUND=ON -DLAPACK_LIBRARIES="-lsci_gnu_mpi_mp" \

.ci/daint.cscs.ch/intel.build.sh

@@ -31,7 +31,7 @@ cd "${SCRATCH}/${BUILD_TAG}.intel"
 
 cmake \
     -DCMAKE_SYSTEM_NAME=CrayLinuxEnvironment \
-    -DUSE_CUDA=ON \
+    -DUSE_ACCEL=cuda \
     -DWITH_GPU=P100 \
     -DBLAS_FOUND=ON -DBLAS_LIBRARIES="-lsci_intel_mpi_mp" \
     -DLAPACK_FOUND=ON -DLAPACK_LIBRARIES="-lsci_intel_mpi_mp" \

.github/workflows/testing-linux.yml

@@ -101,7 +101,7 @@ jobs:
         cmake -G Ninja \
           -DCMAKE_BUILD_TYPE=Release \
           -DUSE_${{ matrix.use_openmp }} \
-          -DUSE_HIP=ON \
+          -DUSE_ACCEL=hip \
           -DWITH_GPU=Mi50 \
           ..
     - name: Build

CMakeLists.txt

@@ -90,15 +90,10 @@ set(USE_SMM
           "Small Matrix Multiplication implementation to use (default: blas)")
 set_property(CACHE USE_SMM PROPERTY STRINGS blas libxsmm)
 
-option(USE_CUDA "Build with CUDA support" OFF)
-option(USE_HIP "Build with HIP support" OFF)
-# USE_CUDA and USE_HIP are mutually exclusive options: we either compile with
-# nvcc OR with hipcc
-if (USE_CUDA AND USE_HIP)
-  message(
-    FATAL_ERROR
-      "USE_CUDA and USE_HIP options are mutually exclusive. Please choose one.")
-endif ()
+set(USE_ACCEL
+    ""
+    CACHE STRING "Build with acceleration support (default: none)")
+set_property(CACHE USE_ACCEL PROPERTY STRINGS opencl cuda hip)
 
 set(SUPPORTED_CUDA_ARCHITECTURES K20X K40 K80 P100 V100)
 set(SUPPORTED_HIP_ARCHITECTURES Mi50)
@@ -117,21 +112,27 @@ enable_language(Fortran)
 
 if (WITH_C_API AND WITH_EXAMPLES)
   enable_language(CXX)
+  enable_language(C)
 endif ()
 
-# we're always using at least C++11
+# always use at least C++11
 set(CMAKE_CXX_STANDARD 11)
 
 # =================================================================================================
 # PACKAGE DISCOVERY (compiler configuration can impact package discovery)
+find_package(PkgConfig)
 
-# =================================== OpenMP and OpenMP/offload backend
+# =================================== OpenMP
 if (USE_OPENMP)
   find_package(OpenMP REQUIRED)
 endif ()
 
+# =================================== LIBXSMM (rely on pkg-config)
+if ((USE_SMM MATCHES "libxsmm") OR (USE_ACCEL MATCHES "opencl"))
+  pkg_check_modules(LIBXSMM IMPORTED_TARGET GLOBAL libxsmmf)
+endif ()
+
 # =================================== BLAS & LAPACK, PkgConfig
-find_package(PkgConfig)
 find_package(LAPACK REQUIRED) # needed for some of the integrated test routines,
                               # also calls find_package(BLAS)
 
@@ -141,8 +142,7 @@ find_package(LAPACK REQUIRED) # needed for some of the integrated test routines,
 # environment for a python interpreter before searching elsewhere in the system.
 # In CMake <3.15, the system is searched before the virtual environment.
 if (NOT Python_EXECUTABLE)
-  # If the python interpreter isn't specified as a command line option, look for
-  # it:
+  # If the python interpreter is not specified (command line), try finding it:
   find_package(
     Python
     COMPONENTS Interpreter
@@ -185,15 +185,32 @@ endif ()
 if (USE_SMM MATCHES "blas")
   message("-- Using BLAS for Small Matrix Multiplication")
 elseif (USE_SMM MATCHES "libxsmm")
-  # rely on pkg-config in order to link against libxsmm
-  pkg_check_modules(deps REQUIRED IMPORTED_TARGET GLOBAL libxsmmf)
-  message("-- Using libxsmm for Small Matrix Multiplication")
+  if (LIBXSMM_FOUND)
+    message("-- Using libxsmm for Small Matrix Multiplication")
+  else ()
+    message(
+      FATAL_ERROR
+        "LIBXSMM is not found but requested (USE_SMM). "
+        "Please install libxsmm and set PKG_CONFIG_PATH=/path/to/libxsmm/lib")
+  endif ()
 else ()
   message(FATAL_ERROR "Unknown SMM library specified")
 endif ()
 
-# =================================== GPU backend
-if (USE_CUDA OR USE_HIP)
+# =================================== GPU backends
+if (USE_ACCEL MATCHES "opencl")
+  if (NOT LIBXSMM_FOUND)
+    message(
+      FATAL_ERROR
+        "LIBXSMM is not found but it is required for the ACC/OpenCL backend. "
+        "Please install libxsmm and set PKG_CONFIG_PATH=/path/to/libxsmm/lib")
+  endif ()
+
+  find_package(OpenCL REQUIRED)
+  enable_language(C)
+endif ()
+
+if (USE_ACCEL MATCHES "cuda|hip")
   enable_language(CXX)
   set(GPU_ARCH_NUMBER_K20X 35)
   set(GPU_ARCH_NUMBER_K40 35)
@@ -203,8 +220,7 @@ if (USE_CUDA OR USE_HIP)
   set(GPU_ARCH_NUMBER_Mi50 gfx906)
 endif ()
 
-if (USE_CUDA)
-
+if (USE_ACCEL MATCHES "cuda")
   enable_language(CUDA)
   if (CMAKE_CUDA_COMPILER_VERSION LESS 5.5)
     message(FATAL_ERROR "CUDA version >= 5.5 is required.")
@@ -243,7 +259,6 @@ if (USE_CUDA)
   else ()
     message(STATUS "Found cuBLAS: ${CUBLAS}")
   endif ()
-
   if (WITH_CUDA_PROFILING)
     find_library(
       CUDA_NVTOOLSEXT nvToolsExt
@@ -257,8 +272,7 @@ endif ()
 
 # inspired from
 # https://github.com/ROCm-Developer-Tools/HIP/tree/master/samples/2_Cookbook/12_cmake_hip_add_executable
-if (USE_HIP)
-
+if (USE_ACCEL MATCHES "hip")
   # Make sure the GPU required is supported
   list(FIND SUPPORTED_HIP_ARCHITECTURES ${WITH_GPU} GPU_SUPPORTED)
   if (GPU_SUPPORTED EQUAL -1)

cmake/CompilerConfiguration.cmake

@@ -88,3 +88,8 @@ Please open an issue at https://github.com/cp2k/dbcsr/issues with the reported c
   message("-- CMAKE_CXX_COMPILER_ID: " ${CMAKE_CXX_COMPILER_ID})
   message("-- CMAKE_CXX_COMPILER full path: " ${CMAKE_CXX_COMPILER})
 endif ()
+
+# inherit C flags from CXX
+set(CMAKE_C_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
+set(CMAKE_C_FLAGS_COVERAGE ${CMAKE_CXX_FLAGS_COVERAGE})
+set(CMAKE_C_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})

docs/guide/2-user-guide/1-installation/index.md

@@ -58,9 +58,8 @@ make
 -DUSE_MPI=<ON|OFF>
 -DUSE_OPENMP=<ON|OFF>
 -DUSE_SMM=<blas|libxsmm>
--DUSE_CUDA=<OFF|ON>
+-DUSE_ACCEL=<opencl|cuda|hip>
 -DWITH_CUDA_PROFILING=<OFF|ON>
--DUSE_HIP=<OFF|ON>
 -DWITH_C_API=<ON|OFF>
 -DWITH_EXAMPLES=<ON|OFF>
 -DWITH_GPU=<P100|K20X|K40|K80|V100|Mi50>

examples/CMakeLists.txt

@@ -6,7 +6,7 @@ set(DBCSR_PROGRAM_SRCS_CPP dbcsr_example_3.cpp dbcsr_tensor_example_2.cpp)
 # Compile Fortran examples
 foreach (dbcsr_program_src ${DBCSR_PROGRAM_SRCS_FTN})
   get_filename_component(dbcsr_program_name ${dbcsr_program_src} NAME_WE)
-  if (USE_HIP)
+  if (USE_ACCEL MATCHES "hip")
     hip_add_executable(${dbcsr_program_name} ${dbcsr_program_src})
   else ()
     add_executable(${dbcsr_program_name} ${dbcsr_program_src})
@@ -24,7 +24,7 @@ if (WITH_C_API)
   foreach (dbcsr_program_src ${DBCSR_PROGRAM_SRCS_CPP})
     get_filename_component(dbcsr_program_name ${dbcsr_program_src} NAME_WE)
     set(dbcsr_program_name ${dbcsr_program_name}_cpp)
-    if (USE_HIP)
+    if (USE_ACCEL MATCHES "hip")
       hip_add_executable(${dbcsr_program_name} ${dbcsr_program_src})
     else ()
       add_executable(${dbcsr_program_name} ${dbcsr_program_src})

src/CMakeLists.txt

@@ -105,6 +105,10 @@ add_fypp_sources(
   utils/dbcsr_toollib.F
   work/dbcsr_work_operations.F)
 
+set(DBCSR_OPENCL_SRCS
+    acc/opencl/acc_opencl.c acc/opencl/acc_opencl_event.c
+    acc/opencl/acc_opencl_mem.c acc/opencl/acc_opencl_stream.c)
+
 set(DBCSR_CUDA_SRCS
     acc/cuda/acc_cublas.cu
     acc/cuda/acc_cuda.cpp
@@ -141,13 +145,9 @@ add_library(dbcsr ${DBCSR_SRCS})
 set_target_properties(dbcsr PROPERTIES VERSION ${dbcsr_VERSION}
                                        SOVERSION ${dbcsr_APIVERSION})
 
-if (TARGET PkgConfig::deps)
-  target_link_libraries(dbcsr PRIVATE PkgConfig::deps)
-endif ()
-
-if (USE_SMM MATCHES "libxsmm")
-  # linker/include flags are managed by pkg-config (above)
+if (LIBXSMM_FOUND)
   target_compile_definitions(dbcsr PRIVATE __LIBXSMM)
+  target_link_libraries(dbcsr PRIVATE PkgConfig::LIBXSMM)
 endif ()
 
 if (BLAS_LIBRARIES MATCHES "mkl_")
@@ -203,6 +203,25 @@ if (OpenMP_FOUND)
   target_link_libraries(dbcsr PRIVATE OpenMP::OpenMP_Fortran)
 endif ()
 
+# =================================================================================================
+# DBCSR LIBRARY's OPENCL BACKEND
+
+if (USE_ACCEL MATCHES "opencl")
+  target_compile_definitions(dbcsr PRIVATE __DBCSR_ACC)
+  target_link_libraries(dbcsr PRIVATE ${OpenCL_LIBRARY})
+
+  # OpenCL backend
+  set(DBCSR_ACC_SRCS ${DBCSR_OPENCL_SRCS})
+  add_library(acc OBJECT ${DBCSR_ACC_SRCS})
+  target_compile_definitions(acc PRIVATE __OPENCL)
+  # account for DBCSR not calling libsmm_acc_init() (DBCSR only calls acc_init)
+  target_compile_definitions(acc PRIVATE __DBCSR_ACC)
+  target_include_directories(acc PRIVATE ${OpenCL_INCLUDE_DIRS})
+  target_sources(dbcsr PRIVATE $<TARGET_OBJECTS:acc>)
+  add_subdirectory(acc/opencl/smm)
+  target_sources(dbcsr PRIVATE $<TARGET_OBJECTS:libsmm_acc>)
+endif ()
+
 # =================================================================================================
 # DBCSR LIBRARY's CUDA BACKEND
 
@@ -240,7 +259,7 @@ function (CUDA_CONVERT_FLAGS EXISTING_TARGET)
   )
 endfunction ()
 
-if (USE_CUDA)
+if (USE_ACCEL MATCHES "cuda")
   if (${CMAKE_VERSION} VERSION_LESS 3.16)
     # workaround for CUDA support with CMake <3.16, see also see
     # https://gitlab.kitware.com/cmake/cmake/issues/17929 and
@@ -296,8 +315,7 @@ endif ()
 # =================================================================================================
 # DBCSR LIBRARY's HIP BACKEND
 
-if (USE_HIP)
-
+if (USE_ACCEL MATCHES "hip")
   if (USE_OPENMP)
     set(HIP_HIPCC_FLAGS "${HIP_HIPCC_FLAGS} ${OpenMP_CXX_FLAGS}")
   endif ()
@@ -335,7 +353,6 @@ if (USE_HIP)
 
   target_compile_definitions(dbcsr PRIVATE __DBCSR_ACC)
   target_compile_definitions(dbcsr PRIVATE __HIP)
-
 endif ()
 
 # =================================================================================================
@@ -401,7 +418,7 @@ write_basic_package_version_file(
   "${CMAKE_CURRENT_BINARY_DIR}/DBCSRConfigVersion.cmake"
   VERSION "${dbcsr_VERSION}"
   COMPATIBILITY SameMajorVersion)
-if (USE_HIP)
+if (USE_ACCEL MATCHES "hip")
   install(
     EXPORT libsmm_accTargets
     NAMESPACE "${config_namespace}"

src/acc/PACKAGE

@@ -1,5 +1,5 @@
 {
 "description": "Generic accelerator API",
 "archive": "libdbcsr",
-"requires": ["../base", "cuda", "hip", "libsmm_acc"]
+"requires": ["../base", "cuda", "hip", "opencl", "libsmm_acc"]
 }