Skip to content

Commit

Permalink
Merge pull request #443 from FESOM/refactoring_levante_shell
Browse files Browse the repository at this point in the history
extend  configure build system to take CMAKE options in commandline and add frame work to directly use multiple shells for a machine
  • Loading branch information
koldunovn authored Mar 27, 2023
2 parents 7d2638d + 1692634 commit ae8d2b7
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 45 deletions.
4 changes: 3 additions & 1 deletion configure.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,7 @@ set -e
source env.sh # source this from your run script too
mkdir build || true # make sure not to commit this to svn or git
cd build
cmake .. -DCMAKE_BUILD_TYPE=Debug # not required when re-compiling
cmake .. $@ -DCMAKE_BUILD_TYPE=Debug # not required when re-compiling
# additional cmake arguments can be passed to configure.sh
# this also includes fesom specific options in CMakeLists, can be used as -DFESOM_COUPLED=ON
make install -j`nproc --all`
28 changes: 21 additions & 7 deletions env.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,18 +17,24 @@ else
BEING_EXECUTED=false
fi

# if an arg is given, use it as hostname
if [ -z "$1" ]; then
# if an arg is given and doesn't start with - use it as hostname, arguments with - are passed on to cmake
if [[ ! -z "$1" ]] && [[ ! "$1" = ^- ]]; then
LOGINHOST=$1 # arg exists and doesn't start with -
shift # pop the argument as we already stored it
else
# no argument given
LOGINHOST="$(hostname -f)"
else
LOGINHOST=$1
fi

if [[ $LOGINHOST =~ ^m[A-Za-z0-9]+\.hpc\.dkrz\.de$ ]]; then
STRATEGY="mistral.dkrz.de"
elif [[ $LOGINHOST =~ ^l[A-Za-z0-9]+\.lvt\.dkrz\.de$ ]]; then
elif [[ $LOGINHOST =~ ^levante ]] || [[ $LOGINHOST =~ ^l[:alnum:]+\.lvt\.dkrz\.de$ ]]; then
STRATEGY="levante.dkrz.de"
# following regex only matches if input is 2 word like levante.nvhpc, this enables using different shells for a machine directly
compid_regex="^([[:alnum:]]+)\.([[:alnum:]]+)$"
if [[ $LOGINHOST =~ $compid_regex ]]; then
COMPILERID="${BASH_REMATCH[2]}"
fi
elif [[ $LOGINHOST =~ ^ollie[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then
STRATEGY="ollie"
elif [[ $LOGINHOST =~ ^albedo[0-9]$ ]] || [[ $LOGINHOST =~ ^prod-[0-9]{4}$ ]]; then
Expand Down Expand Up @@ -74,10 +80,18 @@ fi
DIR="$( cd "$( dirname "${SOURCE}" )" && pwd )"

if [ $BEING_EXECUTED = true ]; then
# file is being executed
# file is being executed, why is this here?
echo $DIR/env/$STRATEGY
else
# file is being sourced
export FESOM_PLATFORM_STRATEGY=$STRATEGY
source $DIR/env/$STRATEGY/shell
SHELLFILE="${DIR}/env/${STRATEGY}/shell"
if [[ -n ${COMPILERID} ]]; then
SHELLFILE="${SHELLFILE}.${COMPILERID}"
fi
if [[ ! -e ${SHELLFILE} ]]; then
echo "Shell file for ${LOGINHOST} doesnt exist: "$SHELLFILE
exit 1
fi
source $SHELLFILE
fi
37 changes: 0 additions & 37 deletions env/levante.dkrz.de/shell

This file was deleted.

1 change: 1 addition & 0 deletions env/levante.dkrz.de/shell
40 changes: 40 additions & 0 deletions env/levante.dkrz.de/shell.gnu
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# make the contents as shell agnostic as possible so we can include them with bash, zsh and others
export LC_ALL=en_US.UTF-8

module load git
module load gcc/11.2.0-gcc-11.2.0

# both mpi below work
#module load intel-oneapi-mpi/2021.5.0-gcc-11.2.0
module load openmpi/4.1.2-gcc-11.2.0

# both below work not sure whats the diff?
module load netcdf-c/4.8.1-intel-oneapi-mpi-2021.5.0-gcc-11.2.0
module load netcdf-fortran/4.5.3-intel-oneapi-mpi-2021.5.0-gcc-11.2.0

#module load netcdf-c/4.8.1-gcc-11.2.0
#module load netcdf-fortran/4.5.3-gcc-11.2.0


export FC=mpif90 CC=mpicc CXX=mpicxx

#module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0
# so use the LD_LIBRARY_PATH or other paths like prefix paths etc for cmake
#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH
spack load intel-oneapi-mkl@2022.0.1%gcc@11.2.0

export OMPI_MCA_pml="ucx"
export OMPI_MCA_btl=self
export OMPI_MCA_osc="pt2pt"
export UCX_IB_ADDR_TYPE=ib_global
# for most runs one may or may not want to disable HCOLL
export OMPI_MCA_coll="^ml,hcoll"
export OMPI_MCA_coll_hcoll_enable="0"
export HCOLL_ENABLE_MCAST_ALL="0"
export HCOLL_MAIN_IB=mlx5_0:1
export UCX_NET_DEVICES=mlx5_0:1
export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
export UCX_UNIFIED_MODE=y
export HDF5_USE_FILE_LOCKING=FALSE
export OMPI_MCA_io="romio321"
export UCX_HANDLE_ERRORS=bt
34 changes: 34 additions & 0 deletions env/levante.dkrz.de/shell.intel
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# make the contents as shell agnostic as possible so we can include them with bash, zsh and others
export LC_ALL=en_US.UTF-8
export CPU_MODEL=AMD_EPYC_ZEN3

module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0
module load openmpi/4.1.2-intel-2021.5.0
export FC=mpif90 CC=mpicc CXX=mpicxx ;
spack load [email protected]%[email protected] # this handles adding to path elegantly then using hardcoded path below
#module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0
#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH

module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0
module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0
module load git # to be able to determine the fesom git SHA when compiling

ulimit -s unlimited # without setting the stack size we get a segfault from the levante netcdf library at runtime
ulimit -c 0 # do not create a coredump after a crash

# environment for Open MPI 4.0.0 and later from https://docs.dkrz.de/doc/levante/running-jobs/runtime-settings.html
export OMPI_MCA_pml="ucx"
export OMPI_MCA_btl=self
export OMPI_MCA_osc="pt2pt"
export UCX_IB_ADDR_TYPE=ib_global
# for most runs one may or may not want to disable HCOLL
export OMPI_MCA_coll="^ml,hcoll"
export OMPI_MCA_coll_hcoll_enable="0"
export HCOLL_ENABLE_MCAST_ALL="0"
export HCOLL_MAIN_IB=mlx5_0:1
export UCX_NET_DEVICES=mlx5_0:1
export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self
export UCX_UNIFIED_MODE=y
export HDF5_USE_FILE_LOCKING=FALSE
export OMPI_MCA_io="romio321"
export UCX_HANDLE_ERRORS=bt

0 comments on commit ae8d2b7

Please sign in to comment.