Skip to content

Commit

Permalink
Amdgpu support (#63)
Browse files Browse the repository at this point in the history 
* Update README.md with zenodo link

* Add makefile for building for AMD GPUs with hip

* Add simple build docs for AMD GPUs

* Update docs/run_amd_gpus.rst

Co-authored-by: Julie Prestopnik <[email protected]>

* Update docs/run_amd_gpus.rst

Co-authored-by: Julie Prestopnik <[email protected]>

---------

Co-authored-by: Julie Prestopnik <[email protected]>
  • Loading branch information
@fluidnumerics-joe @jprestop
fluidnumerics-joe and jprestop authored Jan 27, 2025
1 parent 2e91db3 commit 0ab006f
Show file tree
Hide file tree
Showing 3 changed files with 251 additions and 0 deletions.
213 changes: 213 additions & 0 deletions SRC/FEMAIN/Makefile.hip
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
################################################################################
# FastEddy®: SRC/FEMAIN/Makefile
# ©2016 University Corporation for Atmospheric Research
#
# This file is licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ################################################################################
#
# ################################################################################
# # Available build targets
# ################################################################################

TARGETS = \
FastEddy


################################################################################
## Compiler and flags
################################################################################

TEST_CC = mpicc
TEST_CU_CC = hipcc -Rpass-analysis=kernel-resource-usage --gpu-max-threads-per-block=256
ROCM_PATH ?= /opt/rocm
MPI_ROOT ?= ""
NETCDF_C_ROOT ?= ""
GPU_ARCH ?= gfx90a

# For compilation on NCAR's Casper or Derecho supercomputers use the OTHER_INCLUDES below
OTHER_INCLUDES = -I${MPI_ROOT}/include

DEBUG_CFLAGS = -g

DEFINES = -DCUB_IGNORE_DEPRECATED_CPP_DIALECT -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT

TEST_CFLAGS = -Wall -m64 $(shell ${ROCM_PATH}/bin/hipconfig --cpp_config) ${DEFINES} ${INCLUDES} ${OTHER_INCLUDES}
ARCH_CU_FLAGS = --offload-arch=${GPU_ARCH}
TEST_CU_CFLAGS = ${ARCH_CU_FLAGS} -m64 -std=c++14 ${DEFINES} ${INCLUDES} ${OTHER_INCLUDES}

L_CPPFLAGS =


TEST_LDFLAGS = -L.
TEST_CU_LDFLAGS = -L. -fgpu-rdc

TEST_LIBS = -lm -L${MPI_ROOT}/lib -lmpi -lstdc++ -L${ROCM_PATH}/lib -lamdhip64 -lhiprand -L${NETCDF_C_ROOT}/lib -lnetcdf
TEST_CU_LIBS = -lmpi -L${ROCM_PATH}/lib -lhiprtc


################################################################################
# Sub-directories to INCLUDE
# ################################################################################

INCLUDES = \
-I. \
-I../TIME_INTEGRATION/ \
-I../TIME_INTEGRATION/CUDA \
-I../HYDRO_CORE \
-I../HYDRO_CORE/CUDA \
-I../GRID \
-I../GRID/CUDA \
-I../FECUDA \
-I../IO/ \
-I../MEM_UTILS/ \
-I../FEMPI/ \
-I../PARAMETERS/

################################################################################
# Headers
# ###############################################################################

TEST_HDRS = \
../TIME_INTEGRATION/time_integration.h \
../TIME_INTEGRATION/CUDA/cuda_timeInt.h \
../TIME_INTEGRATION/CUDA/cuda_timeIntDevice_cu.h \
../HYDRO_CORE/hydro_core.h \
../HYDRO_CORE/CUDA/cuda_hydroCore.h \
../HYDRO_CORE/CUDA/cuda_hydroCoreDevice_cu.h \
../GRID/grid.h \
../GRID/CUDA/cuda_grid.h \
../GRID/CUDA/cuda_gridDevice_cu.h \
../FECUDA/fecuda.h \
../FECUDA/fecuda_Device_cu.h \
../IO/io.h \
../MEM_UTILS/mem_utils.h \
../FEMPI/fempi.h \
../PARAMETERS/parameters.h \
../PARAMETERS/hashTable.h

################################################################################
# Objects
# ###############################################################################

TEST_LIB_OBJS =

TEST_OBJS = ./FastEddy.o \
../TIME_INTEGRATION/time_integration.o \
../TIME_INTEGRATION/CUDA/cuda_timeInt.o \
../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.o \
../HYDRO_CORE/hydro_core.o \
../HYDRO_CORE/CUDA/cuda_hydroCore.o \
../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.o \
../GRID/grid.o \
../GRID/CUDA/cuda_grid.o \
../GRID/CUDA/cuda_gridDevice.o \
../FECUDA/fecuda.o \
../FECUDA/fecuda_Device.o \
../IO/io.o \
../IO/ioVarsList.o \
../MEM_UTILS/mem_utils.o \
../FEMPI/fempi.o \
../PARAMETERS/parameters.o \
../PARAMETERS/hashTable.o

################################################################################
# Targets
# ###############################################################################

all: hipify FastEddy

hipify:
find ../ -name "*.cu" -exec hipify-perl -inplace {} \;
find ../ -name "*.h" -exec hipify-perl -inplace {} \;
find ../ -name "*.prehip" -exec rm {} \;


################################################################################
# Generic compile rules
# ################################################################################
%.o: %.cu
rm -rf ./FastEddy_devlink.o; \
$(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@

.c.o:
${COMP_FLAGS} -c $< -o $@

################################################################################
# Module=specific compile rules
# ################################################################################

../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.o: ../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.cu \
../TIME_INTEGRATION/CUDA/cuda_RKschemes.cu
rm -rf ./FastEddy_devlink.o; \
$(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@

../IO/io.o: ../IO/io.c \
../IO/io_netcdf.c \
../IO/io_binary.c
$(COMP_FLAGS) -c $< -o $@

../FECUDA/fecuda_Device.o: ../FECUDA/fecuda_Device.cu \
../FECUDA/fecuda_Utils.cu \
../FECUDA/fecuda_PlugIns.cu
rm -rf ./FastEddy_devlink.o; \
$(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@

../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.o: ../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.cu \
../HYDRO_CORE/CUDA/cuda_BaseStateDevice.cu \
../HYDRO_CORE/CUDA/cuda_buoyancyDevice.cu \
../HYDRO_CORE/CUDA/cuda_coriolisDevice.cu \
../HYDRO_CORE/CUDA/cuda_pressureDevice.cu \
../HYDRO_CORE/CUDA/cuda_BCsDevice.cu \
../HYDRO_CORE/CUDA/cuda_rayleighDampingDevice.cu \
../HYDRO_CORE/CUDA/cuda_advectionDevice.cu \
../HYDRO_CORE/CUDA/cuda_molecularDiffDevice.cu \
../HYDRO_CORE/CUDA/cuda_surfaceLayerDevice.cu \
../HYDRO_CORE/CUDA/cuda_sgsTurbDevice.cu \
../HYDRO_CORE/CUDA/cuda_sgstkeDevice.cu \
../HYDRO_CORE/CUDA/cuda_canopyDevice.cu \
../HYDRO_CORE/CUDA/cuda_largeScaleForcingsDevice.cu \
../HYDRO_CORE/CUDA/cuda_moistureDevice.cu \
../HYDRO_CORE/CUDA/cuda_filtersDevice.cu
rm -rf ./FastEddy_devlink.o; \
$(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@
################################################################################
# Generic Executable
# ###############################################################################

FastEddy: COMP_FLAGS = ${TEST_CC} ${TEST_CFLAGS}

TEST_DEPENDENCIES = ${TEST_OBJS} ${TEST_HDRS} FastEddy_devlink.o

FastEddy_devlink.o:
$(TEST_CU_CC) ${ARCH_CU_FLAGS} ${TEST_OBJS} ${TEST_LDFLAGS} ${TEST_LIBS} -fgpu-rdc --hip-link --emit-static-lib -o FastEddy_devlink.o
FastEddy: ${TEST_DEPENDENCIES}
${COMP_FLAGS} -o FastEddy ${TEST_OBJS} ./FastEddy_devlink.o ${TEST_LDFLAGS} ${TEST_LIBS}

################################################################################
# Clean
# ################################################################################

clean:
rm -rf ${TARGETS} ${TEST_OBJS} ${TEST_LIB_OBJS} ./FastEddy_devlink.o

################################################################################
# Check
# ###############################################################################

check:
@echo CC = ${CC}
@echo CFLAGS = ${CFLAGS}
@echo LDFLAGS = ${LDFLAGS}
@echo LIBS = ${LIBS}
@echo DEPENDENCIES = ${TEST_DEPENDENCIES}
1 change: 1 addition & 0 deletions docs/index.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ FastEddy should be cited as shown here:

release_notes.rst
run_ncar_hpcs.rst
run_amd_gpus.rst
downloads.rst
Tutorials/index
publications.rst
37 changes: 37 additions & 0 deletions docs/run_amd_gpus.rst
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
.. _run_fasteddy_amdgpu:

**************************************************
Building FastEddy on AMD GPU Accelerated platforms
**************************************************

These instructions will help users get started with building FastEddy for systems with AMD GPU accelerators. This can be beneficial for users who have allocations on

* `Oak Ridge National Laboratory's Frontier <https://www.olcf.ornl.gov/frontier/>`_
* `Pawsey Supercomputing Centre's Setonix <https://pawsey.org.au/systems/setonix/>`_
* `CSCS's Lumi <https://www.lumi-supercomputer.eu/may-we-introduce-lumi/>_`
* `Fluid Numerics' Galapagos <https://galapagos.fluidnumerics.com>`_


Compilation
===========

FastEddy requires a C-compiler, MPI, and CUDA.

1. Download the source code from the `Releases <https://github.com/NCAR/FastEddy-model/releases>`_ page and unpack the release in the desired location or clone the `repository <https://github.com/NCAR/FastEddy-model>`_ in the desired location.

2. Navigate to the **SRC/FEMAIN** directory.

3. To build the FastEddy executable run :code:`make -f Makfile.hip` (optionally run :code:`make clean` first if appropriate).

You may need to define a few environment variables that influence the build process to properly set the paths to various dependencies and to select the target GPU.

* :code:`ROCM_PATH` : This is the path to your ROCm installation. This variable defaults to :code:`/opt/rocm`. However, on some systems, multiple versions of ROCm may be available via environment modules and this variable may need to be adjusted accordingly.
* :code:`MPI_ROOT` : This is the path to your MPI installation. Since this environment variable is not necessarily defined through an HPC center's environment modules, it is recommended that you set this variable appropriately.
* :code:`NETCDF_C_ROOT`: This is the path to your NetCDF-C installation. Since this environment variable is not necessarily defined through an HPC center's environment modules, it is recommended that you set this variable appropriately.
* :code:`GPU_ARCH` : This is the AMD GPU architecture code for the target GPU you want to build for. This variable defaults to :code:`gfx90a`, which corresponds to the MI210, MI250, and MI250X GPUs.

The :code:`FastEddy` executable will be located in the **SRC/FEMAIN** directory. To
build on other HPC systems with NVIDIA GPUs, check for availability of the aformentioned
modules/dependencies. Successful compilation may require modifications to shell environment
variable include or library paths, or alternatively minor adjustments to the include or library
flags in **SRC/FEMAIN/Makefile.hip**.

0 comments on commit 0ab006f

Please sign in to comment.