From 2a4427c88434c98b221ccc636479858102a04c5e Mon Sep 17 00:00:00 2001 From: "Henrique B. R" Date: Tue, 9 Jul 2024 14:53:35 +0100 Subject: [PATCH] Added spack check to build script + linting fixes --- scripts/build-platypus-csd3-ampere.sh | 104 ++++++++++++++------------ 1 file changed, 56 insertions(+), 48 deletions(-) diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh index 6eee1ca8..9830eba7 100755 --- a/scripts/build-platypus-csd3-ampere.sh +++ b/scripts/build-platypus-csd3-ampere.sh @@ -7,8 +7,8 @@ #SBATCH -A ukaea-ap001-GPU #SBATCH --cpus-per-task=32 #SBATCH --gres=gpu:1 -#SBATCH --output=platypus_gpu_build/build.%j.out -#SBATCH --error=platypus_gpu_build/build.%j.err +#SBATCH --output=platypus_gpu_build.%j.out +#SBATCH --error=platypus_gpu_build.%j.err ## WARNING: THIS SCRIPT WILL UNINSTALL ALL SPACK MODULES ASSOCIATED WITH ## THE ARCHITECTURE DEFINED IN THE ARCH VARIABLE. IF YOU DO NOT WISH TO DO @@ -18,7 +18,7 @@ ARCH="linux-rocky8-zen" export compile_cores=32 -load_modules(){ +load_modules() { # Load modules . /etc/profile.d/modules.sh # Leave this line (enables the module command) @@ -29,9 +29,9 @@ load_modules(){ } -set_paths(){ +set_paths() { - USER=`whoami` + USER=$(whoami) BUILD_PREFIX=platypus_gpu BUILD_DIR_NAME=${BUILD_PREFIX}_build @@ -39,37 +39,43 @@ set_paths(){ BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME} echo "Building in ${BUILD_PATH}" - mkdir -p ${BUILD_PATH} || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; } + mkdir -p "${BUILD_PATH}" || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; } - cd ${ROOT_PATH} - . spack/share/spack/setup-env.sh + cd "${BUILD_PATH}" || exit 1 - cd ${BUILD_PATH} +} + +check_spack() { + + cd "${ROOT_PATH}" || exit 1 + + if [ $(command -v spack) ]; then + echo "Spack command detected. Using pre-loaded spack." + elif [ -f ${ROOT_PATH}/spack/share/spack/setup-env.sh ]; then + echo "Spack detected in root directory. Loading." + . spack/share/spack/setup-env.sh + else + echo "No spack detected. Building from source." + git clone --depth=100 https://github.com/spack/spack.git + . spack/share/spack/setup-env.sh + fi } -install_spack_deps(){ +install_spack_deps() { # Cleaning up everything to start with a new environment spack uninstall -ay arch=${ARCH} spack external find cuda@11.7.1 - echo "Installing libfabric..." - spack install libfabric@1.19.0 # 1.19.0 is the latest libfabric version that works on CSD3's ampere nodes - spack load libfabric arch=${ARCH} - - echo "Installing hypre..." - spack install hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist \ - ^mpich +cuda cuda_arch=80 \ - ^superlu-dist +cuda cuda_arch=80 +parmetis - spack load hypre arch=${ARCH} - echo "Installing Petsc..." - # spack's petsc doesn't like openmpi, but it works with mpich + # Spack's petsc doesn't like openmpi, but it works with mpich spack install petsc +cuda cuda_arch=80 +fortran +hdf5 +hypre +metis +mpi \ - ^mpich +cuda cuda_arch=80 \ - ^hdf5 +cxx +fortran +hl +mpi +shared + ^mpich +cuda cuda_arch=80 \ + ^hdf5 +cxx +fortran +hl +mpi +shared \ + ^hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist +cublas +gpu-aware-mpi \ + ^superlu-dist +cuda cuda_arch=80 +parmetis +shared spack load petsc arch=${ARCH} echo "Installing SLEPc..." @@ -101,16 +107,16 @@ install_spack_deps(){ } -install_gslib(){ +install_gslib() { echo "Installing gslib..." - cd ${BUILD_PATH} + cd "${BUILD_PATH}" || exit 1 git clone https://github.com/Nek5000/gslib.git - cd gslib + cd gslib || exit 1 make CC=mpicc CFLAGS="-O2 -fPIC" -j"$compile_cores" } -install_mfem(){ +install_mfem() { export CXX=mpic++ export CC=mpicc @@ -119,12 +125,13 @@ install_mfem(){ export FC=mpif90 # Build MFEM - cd ${BUILD_PATH} - git clone https://github.com/Heinrich-BR/mfem.git - cd mfem - git checkout master + cd "${BUILD_PATH}" || exit 1 + git clone https://github.com/mfem/mfem.git + cd mfem || exit 1 + # This is just until MFEM merges Edward's changes. Without this, GPU build crashes! + git checkout EdwardPalmer99/add-missing-header-to-exodus-writer-fix mkdir build - cd build + cd build || exit 1 echo "Building MFEM" cmake .. \ -DCMAKE_BUILD_TYPE=Release \ @@ -132,7 +139,7 @@ install_mfem(){ -DMFEM_USE_OPENMP=NO \ -DMFEM_THREAD_SAFE=YES \ -DMFEM_ENABLE_EXAMPLES=YES \ - -DMFEM_ENABLE_MINIAPPS=YES \ + -DMFEM_ENABLE_MINIAPPS=YES \ -DMFEM_USE_MPI=YES \ -DMFEM_USE_CUDA=YES \ -DCUDA_ARCH=sm_80 \ @@ -140,7 +147,7 @@ install_mfem(){ -DMFEM_USE_SUPERLU=YES \ -DMFEM_USE_NETCDF=YES \ -DMFEM_USE_GSLIB=YES \ - -DGSLIB_DIR=${BUILD_PATH}/gslib/build + -DGSLIB_DIR="${BUILD_PATH}/gslib/build" if [ $? -eq 2 ]; then echo "MFEM config failed" @@ -157,18 +164,18 @@ install_mfem(){ LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${BUILD_PATH}/mfem/build:${BUILD_PATH}/mfem/build/miniapps/common } -install_moose(){ +install_moose() { # Some of the variables needed export MOOSE_JOBS=$compile_cores export LIBMESH_JOBS=$compile_cores export METHOD="opt" - export SLEPC_DIR=`spack find --format "{prefix}" slepc arch=${ARCH}` + export SLEPC_DIR=$(spack find --format "{prefix}" slepc arch=${ARCH}) - cd ${BUILD_PATH} + cd "${BUILD_PATH}" || exit 1 git clone https://github.com/idaholab/moose - cd moose + cd moose || exit 1 echo "Building libmesh..." ./scripts/update_and_rebuild_libmesh.sh --with-mpi @@ -190,14 +197,14 @@ install_moose(){ exit 1 fi - cd framework + cd framework || exit 1 make -j"$compile_cores" if [ $? -eq 2 ]; then echo "MOOSE framework build failed" exit 1 fi - cd ../modules + cd ../modules || exit 1 make -j"$compile_cores" if [ $? -eq 2 ]; then echo "MOOSE modules build failed" @@ -205,7 +212,7 @@ install_moose(){ fi # This takes very long! Only run the tests if you really need to! - #cd ../test + #cd ../test || exit 1 #make -j"$compile_cores" #if [ $? -eq 2 ]; then # echo "MOOSE test build failed" @@ -215,26 +222,27 @@ install_moose(){ #./run_tests -j"$compile_cores" } -install_platypus(){ +install_platypus() { - cd ${BUILD_PATH} + cd "${BUILD_PATH}" || exit 1 echo "Building platypus..." git clone https://github.com/aurora-multiphysics/platypus.git - cd platypus + cd platypus || exit 1 git submodule update --init --recursive - cd contrib/hephaestus/ + cd contrib/hephaestus/ || exit 1 mkdir build - cd build - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR=${BUILD_PATH}/mfem/build .. + cd build || exit 1 + cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR="${BUILD_PATH}/mfem/build" .. ninja - cd ${BUILD_PATH}/platypus + cd ${BUILD_PATH}/platypus || exit 1 make -j"$compile_cores" } load_modules set_paths +check_spack install_spack_deps install_gslib install_mfem