-
Notifications
You must be signed in to change notification settings - Fork 52
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #443 from FESOM/refactoring_levante_shell
extend configure build system to take CMAKE options in commandline and add frame work to directly use multiple shells for a machine
- Loading branch information
Showing
5 changed files
with
99 additions
and
45 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
shell.intel |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
# make the contents as shell agnostic as possible so we can include them with bash, zsh and others | ||
export LC_ALL=en_US.UTF-8 | ||
|
||
module load git | ||
module load gcc/11.2.0-gcc-11.2.0 | ||
|
||
# both mpi below work | ||
#module load intel-oneapi-mpi/2021.5.0-gcc-11.2.0 | ||
module load openmpi/4.1.2-gcc-11.2.0 | ||
|
||
# both below work not sure whats the diff? | ||
module load netcdf-c/4.8.1-intel-oneapi-mpi-2021.5.0-gcc-11.2.0 | ||
module load netcdf-fortran/4.5.3-intel-oneapi-mpi-2021.5.0-gcc-11.2.0 | ||
|
||
#module load netcdf-c/4.8.1-gcc-11.2.0 | ||
#module load netcdf-fortran/4.5.3-gcc-11.2.0 | ||
|
||
|
||
export FC=mpif90 CC=mpicc CXX=mpicxx | ||
|
||
#module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0 | ||
# so use the LD_LIBRARY_PATH or other paths like prefix paths etc for cmake | ||
#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH | ||
spack load intel-oneapi-mkl@2022.0.1%gcc@11.2.0 | ||
|
||
export OMPI_MCA_pml="ucx" | ||
export OMPI_MCA_btl=self | ||
export OMPI_MCA_osc="pt2pt" | ||
export UCX_IB_ADDR_TYPE=ib_global | ||
# for most runs one may or may not want to disable HCOLL | ||
export OMPI_MCA_coll="^ml,hcoll" | ||
export OMPI_MCA_coll_hcoll_enable="0" | ||
export HCOLL_ENABLE_MCAST_ALL="0" | ||
export HCOLL_MAIN_IB=mlx5_0:1 | ||
export UCX_NET_DEVICES=mlx5_0:1 | ||
export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self | ||
export UCX_UNIFIED_MODE=y | ||
export HDF5_USE_FILE_LOCKING=FALSE | ||
export OMPI_MCA_io="romio321" | ||
export UCX_HANDLE_ERRORS=bt |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
# make the contents as shell agnostic as possible so we can include them with bash, zsh and others | ||
export LC_ALL=en_US.UTF-8 | ||
export CPU_MODEL=AMD_EPYC_ZEN3 | ||
|
||
module load intel-oneapi-compilers/2022.0.1-gcc-11.2.0 | ||
module load openmpi/4.1.2-intel-2021.5.0 | ||
export FC=mpif90 CC=mpicc CXX=mpicxx ; | ||
spack load [email protected]%[email protected] # this handles adding to path elegantly then using hardcoded path below | ||
#module load intel-oneapi-mkl/2022.0.1-gcc-11.2.0 | ||
#export LD_LIBRARY_PATH=/sw/spack-levante/intel-oneapi-mkl-2022.0.1-ttdktf/mkl/2022.0.1/lib/intel64:$LD_LIBRARY_PATH | ||
|
||
module load netcdf-c/4.8.1-openmpi-4.1.2-intel-2021.5.0 | ||
module load netcdf-fortran/4.5.3-openmpi-4.1.2-intel-2021.5.0 | ||
module load git # to be able to determine the fesom git SHA when compiling | ||
|
||
ulimit -s unlimited # without setting the stack size we get a segfault from the levante netcdf library at runtime | ||
ulimit -c 0 # do not create a coredump after a crash | ||
|
||
# environment for Open MPI 4.0.0 and later from https://docs.dkrz.de/doc/levante/running-jobs/runtime-settings.html | ||
export OMPI_MCA_pml="ucx" | ||
export OMPI_MCA_btl=self | ||
export OMPI_MCA_osc="pt2pt" | ||
export UCX_IB_ADDR_TYPE=ib_global | ||
# for most runs one may or may not want to disable HCOLL | ||
export OMPI_MCA_coll="^ml,hcoll" | ||
export OMPI_MCA_coll_hcoll_enable="0" | ||
export HCOLL_ENABLE_MCAST_ALL="0" | ||
export HCOLL_MAIN_IB=mlx5_0:1 | ||
export UCX_NET_DEVICES=mlx5_0:1 | ||
export UCX_TLS=mm,knem,cma,dc_mlx5,dc_x,self | ||
export UCX_UNIFIED_MODE=y | ||
export HDF5_USE_FILE_LOCKING=FALSE | ||
export OMPI_MCA_io="romio321" | ||
export UCX_HANDLE_ERRORS=bt |