Merge pull request #67 from ecmwf-ifs/naan-cce17

Add CPU only support for CCE17

CMakeLists.txt

@@ -61,7 +61,8 @@ endif()
 ecbuild_add_option( FEATURE ACC
                     DEFAULT ON
                     DESCRIPTION "Support for using GPUs with OpenACC"
-                    REQUIRED_PACKAGES "OpenACC COMPONENTS Fortran" )
+                    REQUIRED_PACKAGES "OpenACC COMPONENTS Fortran" 
+                    CONDITION CMAKE_Fortran_COMPILER_ID MATCHES "PGI|NVHPC")
 
 ## set general compiler flags
 include(cmake/field_api_compile_options.cmake)
@@ -124,6 +125,13 @@ ecbuild_add_option( FEATURE BUDDY_MALLOC
     DEFAULT ON
 )
 
+## Field Gang
+ecbuild_add_option( FEATURE FIELD_GANG
+    DESCRIPTION "Enable packed storage of fields"
+    DEFAULT ON
+    CONDITION (NOT CMAKE_Fortran_COMPILER_ID MATCHES "Cray")
+)
+
 check_symbol_exists(backtrace execinfo.h HAVE_BACKTRACE)
 
 ## fypp preprocessor flags
@@ -136,12 +144,21 @@ endif()
 if(fiat_FOUND)
   list( APPEND fypp_defines "-DWITH_FIAT")
 endif()
+if(HAVE_FIELD_GANG)
+  list( APPEND fypp_defines "-DHAVE_GANG")
+endif()
 
+unset( ranksuff_srcs )
 ## preprocess fypp files
+list( APPEND ranksuff_srcs _shuffle _access _util _array _array_util _factory _gather _data)
+if(HAVE_FIELD_GANG)
+  list( APPEND ranksuff_srcs _gang )
+endif()
+
 foreach (SUFF IN ITEMS IM RM RD LM)
   string (TOLOWER ${SUFF} suff)
   foreach (RANK RANGE 1 5)
-    foreach (FUNC IN ITEMS "" _shuffle _access _util _array _array_util _gang _factory _gather _data)
+    foreach (FUNC ${ranksuff_srcs} "")
       add_custom_command (OUTPUT field_${RANK}${suff}${FUNC}_module.F90
 	      COMMAND ${FYPP} -DRANK=${RANK} -DSUFF='${SUFF}' ${fypp_defines} -m os -M ${CMAKE_CURRENT_SOURCE_DIR} -m fieldType 
         ${CMAKE_CURRENT_SOURCE_DIR}/field_RANKSUFF${FUNC}_module.fypp > field_${RANK}${suff}${FUNC}_module.F90
@@ -153,9 +170,15 @@ foreach (SUFF IN ITEMS IM RM RD LM)
   endforeach ()
 endforeach ()
 
-foreach (SRC IN ITEMS dev_alloc_module field_factory_module field_access_module field_gang_module field_array_module field_module 
+unset( non_ranksuff_srcs )
+list ( APPEND non_ranksuff_srcs dev_alloc_module field_factory_module field_access_module field_array_module field_module 
                       field_shuffle_module field_util_module field_array_util_module field_shuffle_type_module host_alloc_module
                       field_gathscat_module field_gathscat_type_module)
+if(HAVE_FIELD_GANG)
+  list( APPEND non_ranksuff_srcs field_gang_module )
+endif()
+
+foreach (SRC ${non_ranksuff_srcs} )
   add_custom_command (OUTPUT ${SRC}.F90
 	  COMMAND ${FYPP} -m os ${fypp_defines} -M ${CMAKE_CURRENT_SOURCE_DIR} -m fieldType ${CMAKE_CURRENT_SOURCE_DIR}/${SRC}.fypp > ${SRC}.F90
     DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SRC}.fypp

Readme.md

@@ -44,15 +44,17 @@ Features of FIELD_API can be toggled by passing the following argument to the CM
 |:--- |:--- |:--- |
 | TESTS | ON | Build the testing suite. |
 | BUDDY_MALLOC | ON | Enable the use of a binary buddy memory allocator for the shadow host allocation for `FIELD%DEVPTR`. This option is switched off if CUDA is enabled.|
-| ACC | ON | Enable the use of OpenACC for GPU offload. |
+| ACC | ON | Enable the use of OpenACC for GPU offload. Currently only suppored on NVHPC. |
 | SINGLE_PRECISION | ON | Enable the compilation of field_api in single precision |
 | DOUBLE_PRECISION | ON | Enable the compilation of field_api in double precision |
 | CUDA | OFF | Enable the use of CUDA for GPU offload. Disables the use of the buddy memory allocator, removes the shadow host allocation for `FIELD%DEVPTR` and allocates owned fields (see below) in pinned (page-locked) host memory.|
+| FIELD_GANG | ON | Enable packed storage of groups of fields. This feature is not supported for the Cray compiler as it cannot resolve the underlying polymorphism.|
 
 ## Supported compilers
 The library has been tested with the nvhpc toolkit from Nvidia, version 23.9
-and is continually tested with newer releases. It has also been tested on CPU
-(-DENABLE_ACC=OFF) with GCC 12 and Intel 2021. The CI testing (CPU-only for now) uses GNU 11.4.
+and is continually tested with newer releases. Please note that GPU offload is currently
+only supported for Nvidia compilers. It has also been tested on CPU (-DENABLE_ACC=OFF)
+with GCC 12, Intel 2021 and CCE17.
 
 # Field API types
 

arch/eurohpc/lumi/cray-gpu/17.0.1/env.sh

@@ -0,0 +1,71 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+# Source me to get the correct configure/build/run environment
+
+# Store tracing and disable (module is *way* too verbose)
+{ tracing_=${-//[^x]/}; set +x; } 2>/dev/null
+
+module_load() {
+  echo "+ module load $1"
+  module load $1
+}
+module_unload() {
+  echo "+ module unload $1"
+  module unload $1
+}
+
+# Unload to be certain
+module reset
+
+# Load modules
+module_load LUMI/24.03
+module_load partition/G
+module_load PrgEnv-cray/8.4.0
+module_load cce/17.0.1
+# module_load cray-mpich/8.1.27
+module_load cray-mpich/8.1.29
+module_load craype-network-ofi
+module_load rocm/6.0.3
+module_load buildtools/24.03
+# module_load Boost/1.82.0-cpeCray-23.09
+module_load cray-libsci/24.03.0
+module_load Boost/1.83.0-cpeCray-24.03
+module_load cray-python/3.10.10
+module_load craype-x86-trento
+module_load craype-accel-amd-gfx90a
+
+### Handling of "magic" cray modules
+# 1) Load the cray modules
+# module_load cray-hdf5/1.12.2.7
+module_load cray-hdf5/1.12.2.11
+# 2) Store variables to locate the packages
+_HDF5_ROOT=${CRAY_HDF5_PREFIX}
+# 3) Unload the cray modules in reverse order, removing all the magic
+module_unload cray-hdf5
+# 4) Define variables that CMake introspects
+export HDF5_ROOT=${_HDF5_ROOT}
+
+# Export environment variable3s
+export MPI_HOME=${MPICH_DIR}
+export CC=cc
+export CXX=CC
+export FC=ftn
+export HIPCXX=$(hipconfig --hipclangpath)/clang++
+
+module list
+
+set -x
+
+# Restore tracing to stored setting
+{ if [[ -n "$tracing_" ]]; then set -x; else set +x; fi } 2>/dev/null
+
+# export ECBUILD_TOOLCHAIN="./toolchain.cmake"
+path=$BASH_SOURCE
+DIR_PATH=$(dirname $path)
+export ECBUILD_TOOLCHAIN=$DIR_PATH/toolchain.cmake

arch/eurohpc/lumi/cray-gpu/17.0.1/toolchain.cmake

@@ -0,0 +1,52 @@
+# (C) Copyright 1988- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+####################################################################
+# COMPILER
+####################################################################
+
+set( ECBUILD_FIND_MPI OFF )
+set( ENABLE_USE_STMT_FUNC ON CACHE STRING "" )
+
+####################################################################
+# OpenMP FLAGS
+####################################################################
+
+set( OpenMP_C_FLAGS           "-fopenmp" CACHE STRING "" )
+set( OpenMP_CXX_FLAGS         "-fopenmp" CACHE STRING "" )
+set( OpenMP_Fortran_FLAGS     "-homp -hlist=aimd" CACHE STRING "" )
+set( OpenMP_C_LIB_NAMES       "craymp" CACHE STRING "" )
+set( OpenMP_CXX_LIB_NAMES     "craymp" CACHE STRING "" )
+set( OpenMP_Fortran_LIB_NAMES "craymp" CACHE STRING "" )
+set( OpenMP_craymp_LIBRARY    "/opt/cray/pe/cce/17.0.1/cce/x86_64/lib/libcraymp.so" CACHE STRING "" )
+
+####################################################################
+# OpenACC FLAGS
+####################################################################
+
+set( OpenACC_C_FLAGS "-hacc" CACHE STRING "" )
+# set( OpenACC_CXX_FLAGS "-hacc" CACHE STRING "" )
+# set( OpenACC_Fortran_FLAGS "-hacc" CACHE STRING "" )
+
+####################################################################
+# Compiler FLAGS
+####################################################################
+
+# General Flags (add to default)
+set(ECBUILD_Fortran_FLAGS "-h acc_model=auto_async_none")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -hbyteswapio")
+set(ECBUILD_Fortran_FLAGS "${ECBUILD_Fortran_FLAGS} -Wl, --as-needed")
+
+set(ECBUILD_Fortran_FLAGS_BIT "-O3 -hfp1 -hscalar3 -hvector3 -G2 -haggress -DNDEBUG")
+
+if(NOT DEFINED CMAKE_HIP_ARCHITECTURES)
+  set(CMAKE_HIP_ARCHITECTURES gfx90a)
+endif()
+
+# select OpenMP pragma to be used
+# set( HAVE_OMP_TARGET_LOOP_CONSTRUCT_BIND_PARALLEL OFF CACHE BOOL "" )

field_RANKSUFF_data_module.fypp

@@ -28,7 +28,11 @@ PUBLIC :: ${ftn}$_COPY_INTF
 
 ABSTRACT INTERFACE
   SUBROUTINE ${ftn}$_COPY_INTF (HST, DEV, MAP_DEVPTR, KDIR, QUEUE)
+#:if 'IM' in ftn
+    IMPORT :: ${ft.kind}$
+#:else
     IMPORT :: JPIM, ${ft.kind}$
+#:endif
     ${ft.type}$, POINTER :: HST (${ft.shape}$), DEV (${ft.shape}$)
     LOGICAL,                       INTENT (IN) :: MAP_DEVPTR
     INTEGER (KIND=JPIM),           INTENT (IN) :: KDIR

field_RANKSUFF_factory_module.fypp

@@ -13,10 +13,12 @@ MODULE FIELD_${RANK}$${SUFF}$_FACTORY_MODULE
 
 #:for ft in fieldTypeList
 USE ${ft.name}$_MODULE
+#:if defined('HAVE_GANG')
 #:if ft.ganged
 USE FIELD_${ft.rank-1}$${SUFF}$_MODULE
 USE ${ft.name}$_GANG_MODULE
 #:endif
+#:endif
 #:endfor
 
 ${fieldType.useParkind1 ()}$
@@ -29,10 +31,12 @@ INTERFACE FIELD_NEW
 #:for ft in fieldTypeList
   MODULE PROCEDURE ${ft.name}$_NEW_OWNER
   MODULE PROCEDURE ${ft.name}$_NEW_WRAPPER
+#:if defined('HAVE_GANG')
 #:if ft.ganged
   MODULE PROCEDURE ${ft.name}$_NEW_GANG_WRAPPER
   MODULE PROCEDURE ${ft.name}$_NEW_GANG_OWNER
 #:endif
+#:endif
 #:endfor
 END INTERFACE
 
@@ -108,6 +112,7 @@ IF (FIELD_STATISTICS_ENABLE) CALL FIELD_STATISTICS_NEW ()
 
 END SUBROUTINE
 
+#:if defined('HAVE_GANG')
 #:if ft.ganged
 SUBROUTINE ${ft.name}$_NEW_GANG_WRAPPER (FIELD_PTR, CHILDREN, LBOUNDS, PERSISTENT, DATA, SYNC_ON_FINAL, INITIALIZED)
 
@@ -180,6 +185,7 @@ IF (FIELD_STATISTICS_ENABLE) CALL FIELD_STATISTICS_NEW ()
 
 END SUBROUTINE
 
+#:endif
 #:endif
 
 SUBROUTINE ${ft.name}$_DELETE (FIELD_PTR)

field_RANKSUFF_gather_module.fypp

@@ -30,21 +30,14 @@ PUBLIC :: FIELD_SHUFFLE
 
 #:for what in ['DEVICE', 'HOST']
 #:for mode in ['RDONLY', 'RDWR']
-INTERFACE GATHER_${what}$_DATA_${mode}$
+
 #:for ft in fieldTypeList
-  MODULE PROCEDURE :: ${ft.name}$_GATHER_${what}$_DATA_${mode}$
+PUBLIC :: ${ft.name}$_GATHER_${what}$_DATA_${mode}$
 #:endfor
-END INTERFACE
-
-PUBLIC :: GATHER_${what}$_DATA_${mode}$
 
-INTERFACE SGATHER_${what}$_DATA_${mode}$
 #:for ft in fieldTypeList
-  MODULE PROCEDURE :: S${ft.name}$_GATHER_${what}$_DATA_${mode}$
+PUBLIC :: S${ft.name}$_GATHER_${what}$_DATA_${mode}$
 #:endfor
-END INTERFACE
-
-PUBLIC :: SGATHER_${what}$_DATA_${mode}$
 
 #:endfor
 #:endfor

field_gathscat_module.fypp

@@ -9,7 +9,7 @@
 
 MODULE FIELD_GATHSCAT_MODULE
 
-#:set fieldTypeList = fieldType.getFieldTypeList ()
+#:set fieldTypeList = fieldType.getFieldTypeList (hasView=True)
 
 USE FIELD_ACCESS_MODULE
 USE FIELD_FACTORY_MODULE
@@ -30,9 +30,24 @@ PUBLIC :: FIELD_GATHSCAT
 #:for what in ['DEVICE', 'HOST']
 #:for mode in ['RDONLY', 'RDWR']
 
+INTERFACE GATHER_${what}$_DATA_${mode}$
+#:for ft in fieldTypeList
+  MODULE PROCEDURE ${ft.name}$_GATHER_${what}$_DATA_${mode}$
+#:endfor
+END INTERFACE GATHER_${what}$_DATA_${mode}$
+
 PUBLIC :: GATHER_${what}$_DATA_${mode}$
 
+INTERFACE SGATHER_${what}$_DATA_${mode}$
+#:for ft in fieldTypeList
+  MODULE PROCEDURE S${ft.name}$_GATHER_${what}$_DATA_${mode}$
+#:endfor
+END INTERFACE SGATHER_${what}$_DATA_${mode}$
+
+PUBLIC :: SGATHER_${what}$_DATA_${mode}$
+
 #:endfor
 #:endfor
 
+
 END MODULE FIELD_GATHSCAT_MODULE

field_shuffle_module.fypp

@@ -14,11 +14,15 @@ MODULE FIELD_SHUFFLE_MODULE
 USE FIELD_ACCESS_MODULE
 USE FIELD_FACTORY_MODULE
 USE FIELD_SHUFFLE_TYPE_MODULE
+#:for what in ['DEVICE', 'HOST']
+#:for mode in ['RDONLY', 'RDWR']
+USE FIELD_GATHSCAT_MODULE, ONLY : GATHER_${what}$_DATA_${mode}$
+#:endfor
+#:endfor
 ${fieldType.useParkind1 ()}$
 
 #:for ft in fieldTypeList
 USE ${ft.name}$_SHUFFLE_MODULE
-USE ${ft.name}$_GATHER_MODULE
 #:endfor
 
 

host_alloc_module.fypp

@@ -68,7 +68,7 @@ INTERFACE
 END INTERFACE
 
 TYPE :: MEM_BLOCK
-  TYPE(C_PTR) :: DATA
+  TYPE(C_PTR) :: DATA = C_NULL_PTR
   INTEGER(KIND=INT64) :: POS = 0
   INTEGER(KIND=INT64) :: SIZE = 0
   INTEGER :: NUMFLDS = 0