CMake fixes

.github/workflows/compile.yml

@@ -82,10 +82,11 @@ jobs:
         run: |
           if [[ -f /opt/intel/oneapi/setvars.sh ]]; then
             source /opt/intel/oneapi/setvars.sh
+            export FC=mpiifx
           fi
           mkdir build
           cd build
-          cmake --preset=${{ matrix.toolchain }} -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_HYPRE=${{ matrix.hypre }} ../
+          cmake -DCMAKE_BUILD_TYPE=DEBUG -DENABLE_HYPRE=${{ matrix.hypre }} ../
 
       - name : Build
         run: |

.github/workflows/precision.yml

@@ -39,7 +39,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake --preset=gnu \
+          cmake \
             -DENABLE_HYPRE=${{ matrix.hypre }} \
             -DENABLE_FP32_FIELDS=${{ matrix.sp_fields }} \
             -DENABLE_FP32_POIS=${{ matrix.sp_pois }} \

.github/workflows/test.yml

@@ -30,7 +30,7 @@ jobs:
         run: |
           mkdir build
           cd build
-          cmake --preset=gnu -DCMAKE_BUILD_TYPE=Debug ../
+          cmake -DCMAKE_BUILD_TYPE=Debug ../
 
       - name: Build
         run: |

CMakeLists.txt

@@ -33,10 +33,17 @@ ecbuild_add_option( FEATURE DALESLIB
                     DESCRIPTION "Build DALES libraries for use with OMUSE"
                     DEFAULT OFF )
 
+ecbuild_add_option( FEATURE ACC
+                    DESCRIPTION "Build with GPU support through OpenACC"
+                    DEFAULT OFF )
+
 # Check for required dependencies
 
-ecbuild_find_package( NetCDF REQUIRED COMPONENTS Fortran )
+ecbuild_find_package( NetCDF REQUIRED COMPONENTS Fortran C )
+
 ecbuild_find_mpi( COMPONENTS Fortran REQUIRED )
+ecbuild_info( "MPI compiler: ${MPI_Fortran_COMPILER}")
+set ( CMAKE_Fortran_COMPILER ${MPI_Fortran_COMPILER} )
 
 # Set precision
 
@@ -70,8 +77,32 @@ if( ENABLE_HYPRE )
 endif()
 
 if( ENABLE_NVTX )
-  ecbuild_add_fortran_flags( -lnvToolsExt )
   add_compile_definitions( USE_NVTX )
 endif()
 
-add_subdirectory( src )
+# GPU stuff
+
+include( dales_find_cuda )
+
+if( ENABLE_ACC )
+  find_package( OpenACC COMPONENTS Fortran QUIET )
+  if( OpenACC_Fortran_FOUND )
+    set( OpenACC ON )
+    ecbuild_info( "Found OpenACC" )
+  endif()
+
+  # For debug builds, enable accelerator info
+  if( ${CMAKE_Fortran_COMPILER_ID} MATCHES PGI|NVHPC )
+    ecbuild_add_fortran_flags( "-Minfo=accel" BUILD DEBUG ) 
+  endif()
+
+  add_compile_definitions( DALES_GPU )
+endif()
+
+# For GPU builds and/or NVTX profiling, we need the CUDA toolkit
+if( ENABLE_ACC OR ENABLE_NVTX )
+  dales_find_cuda()
+endif()
+
+include( dales_compile_options )
+add_subdirectory( src )

cmake/dales_compile_options.cmake

@@ -0,0 +1,18 @@
+# Compiler flag presets for some frequently used compilers
+if( CMAKE_Fortran_COMPILER_ID MATCHES GNU )
+  ecbuild_add_fortran_flags( "-cpp -W -Wall -Wno-tabs -Wno-compare-reals -fdefault-real-8 -fdefault-double-8 -march=native -ffree-line-length-none -std=gnu -Werror=implicit-interface" )
+  ecbuild_add_fortran_flags( "-funroll-all-loops -fno-f2c -Ofast -g -fbacktrace" BUILD RELEASE )
+  ecbuild_add_fortran_flags( "-finit-real=nan -fbounds-check -fbacktrace -fno-f2c -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG )
+elseif( CMAKE_Fortran_COMPILER_ID MATCHES Intel )
+  ecbuild_add_fortran_flags( "-cpp -r8 -ftz" )
+  ecbuild_add_fortran_flags( "-O3" BUILD RELEASE )
+  ecbuild_add_fortran_flags( "-traceback -fpe1 -O0 -g -check all" BUILD DEBUG )
+elseif( CMAKE_Fortran_COMPILER_ID MATCHES PGI|NVHPC )
+  ecbuild_add_fortran_flags( "-Mpreprocess -Mr8 -Mfree" )
+  ecbuild_add_fortran_flags( "-Ofast" BUILD RELEASE )
+  ecbuild_add_fortran_flags( "-Minit-real=snan -traceback -O0 -g -ffpe-trap=invalid,zero,overflow" BUILD DEBUG )
+elseif( CMAKE_Fortran_COMPILER_ID MATCHES Fujitsu )
+  ecbuild_add_fortran_flags( "-Cpp -CcdRR8 -g -Koptmsg=2" )
+  ecbuild_add_fortran_flags( "-Kfast -x 128" BUILD RELEASE )
+  ecbuild_add_fortran_flags( "-Haefosux -O0" BUILD DEBUG )
+endif()

cmake/dales_find_cuda.cmake

@@ -0,0 +1,21 @@
+macro( dales_find_cuda )
+  # Check if a target architecture has been set. If not, default to 80 (A100 & RTX3090)
+  # For H100, use 90
+  if( NOT DEFINED CMAKE_CUDA_ARCHITECTURES )
+    set( CMAKE_CUDA_ARCHITECTURES 80 )
+  endif()
+
+  # Look for the CUDA Toolkit, which has cuFFT and NVTX
+  find_package( CUDAToolkit )
+  if( NOT TARGET CUDA::cufft AND ENABLE_ACC )
+    ecbuild_error( "Could not find cuFFT, which is required for GPU builds!" )
+  else()
+    ecbuild_info( "Found cuFFT: ${CUDA_cufft_LIBRARY}" )
+  endif()
+  if( NOT TARGET CUDA::nvToolsExt AND ENABLE_NVTX )
+    ecbuild_error( "Could not find NVTX library for profiling" )
+  else()
+    ecbuild_info( "Found NVTX: ${CUDA_nvToolsExt_LIBRARY}" )
+  endif() 
+endmacro()
+

src/CMakeLists.txt

@@ -219,39 +219,54 @@ add_custom_target( tag_git_version ALL
   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90
 )
 
+# Temporary, because we use CUDA-Fortran in these places
+if( ENABLE_ACC )
+  set_source_files_properties( modmpi.f90 modgpumpiinterface.f90 PROPERTIES COMPILE_OPTIONS -cuda )
+endif()
+
 # Stand-alone DALES program
 ecbuild_add_executable( TARGET dales
-												SOURCES
-													${dales_srcs}
-													${rrtmg_lw_srcs}
-													${rrtmg_sw_srcs}
-													${rrtmgp_srcs}
-													${CMAKE_CURRENT_BINARY_DIR}/modversion.f90
-												DEPENDS tag_git_version
-												INCLUDES
-													${NETCDF_INCLUDE_DIR}
-													$<$<BOOL:${ENABLE_FFTW}>:${FFTW_INCLUDE_DIRS}>
-												LIBS
-													NetCDF::NetCDF_Fortran
-													$<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3>
-													$<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3f>
-													$<$<BOOL:${ENABLE_HYPRE}>:${HYPRE_LIB}> )
+						SOURCES
+							${dales_srcs}
+							${rrtmg_lw_srcs}
+							${rrtmg_sw_srcs}
+							${rrtmgp_srcs}
+							${CMAKE_CURRENT_BINARY_DIR}/modversion.f90
+						DEPENDS tag_git_version
+						INCLUDES
+							${NETCDF_INCLUDE_DIR}
+							$<$<BOOL:${ENABLE_FFTW}>:${FFTW_INCLUDE_DIRS}>
+						LIBS
+							NetCDF::NetCDF_Fortran
+                            NetCDF::NetCDF_C
+                            $<$<BOOL:${ENABLE_ACC}>:OpenACC::OpenACC_Fortran>
+                            $<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3>
+                            $<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3f>
+                            $<$<BOOL:${ENABLE_HYPRE}>:${HYPRE_LIB}> 
+                            $<$<BOOL:${ENABLE_ACC}>:CUDA::cufft> 
+                            $<$<BOOL:${ENABLE_NVTX}>:CUDA::nvToolsExt>
+)
+# Temporary
+if( ENABLE_ACC )
+  target_link_options( dales PUBLIC -cuda )
+endif()
 
 # DALES library, e.g. for use with OMUSE. Only built if -DENABLE_DALESLIB is turned on.
 ecbuild_add_library( TARGET libdales
-										 SOURCES
-												${dales_srcs}
-												${rrtmg_lw_srcs}
-												${rrtmg_sw_srcs}
-												${rrtmgp_srcs}
-												${CMAKE_CURRENT_BINARY_DIR}/modversion.f90
-										 DEPENDS tag_git_version
-										 PUBLIC_INCLUDES
-										   ${NETCDF_INCLUDE_DIR}
-											 $<$<BOOL:${ENABLE_FFTW}>:${FFTW_INCLUDE_DIRS}>
-										 PUBLIC_LIBS
-										   NetCDF::NetCDF_Fortran
-											 $<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3>
-											 $<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3f>
-											 $<$<BOOL:${ENABLE_HYPRE}>:${HYPRE_LIB}>
-										 CONDITION ENABLE_DALESLIB )
+					 SOURCES
+     				   ${dales_srcs}
+					   ${rrtmg_lw_srcs}
+				       ${rrtmg_sw_srcs}
+					   ${rrtmgp_srcs}
+					   ${CMAKE_CURRENT_BINARY_DIR}/modversion.f90
+					 DEPENDS tag_git_version
+					 PUBLIC_INCLUDES
+					   ${NETCDF_INCLUDE_DIR}
+					   $<$<BOOL:${ENABLE_FFTW}>:${FFTW_INCLUDE_DIRS}>
+					 PUBLIC_LIBS
+					   NetCDF::NetCDF_Fortran
+					   $<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3>
+     				   $<$<BOOL:${ENABLE_FFTW}>:FFTW::fftw3f>
+					   $<$<BOOL:${ENABLE_HYPRE}>:${HYPRE_LIB}>
+					 CONDITION ENABLE_DALESLIB
+)