Added spack check to build script + linting fixes

scripts/build-platypus-csd3-ampere.sh

@@ -7,8 +7,8 @@
 #SBATCH -A ukaea-ap001-GPU
 #SBATCH --cpus-per-task=32
 #SBATCH --gres=gpu:1
-#SBATCH --output=platypus_gpu_build/build.%j.out
-#SBATCH --error=platypus_gpu_build/build.%j.err
+#SBATCH --output=platypus_gpu_build.%j.out
+#SBATCH --error=platypus_gpu_build.%j.err
 
 ## WARNING: THIS SCRIPT WILL UNINSTALL ALL SPACK MODULES ASSOCIATED WITH 
 ## THE ARCHITECTURE DEFINED IN THE ARCH VARIABLE. IF YOU DO NOT WISH TO DO 
@@ -18,7 +18,7 @@ ARCH="linux-rocky8-zen"
 
 export compile_cores=32
 
-load_modules(){
+load_modules() {
 
     # Load modules
     . /etc/profile.d/modules.sh                # Leave this line (enables the module command)
@@ -29,47 +29,53 @@ load_modules(){
 
 }
 
-set_paths(){
+set_paths() {
 
-    USER=`whoami`
+    USER=$(whoami)
     BUILD_PREFIX=platypus_gpu
     BUILD_DIR_NAME=${BUILD_PREFIX}_build
 
     ROOT_PATH=/home/${USER}/rds/rds-ukaea-ap001/${USER}
     BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME}
 
     echo "Building in ${BUILD_PATH}"
-    mkdir -p ${BUILD_PATH} || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
+    mkdir -p "${BUILD_PATH}" || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
 
-    cd ${ROOT_PATH}
-    . spack/share/spack/setup-env.sh
+    cd "${BUILD_PATH}" || exit 1
 
-    cd ${BUILD_PATH}
+}
+
+check_spack() {
+
+    cd "${ROOT_PATH}" || exit 1
+
+    if [ $(command -v spack) ]; then
+        echo "Spack command detected. Using pre-loaded spack."
+    elif [ -f ${ROOT_PATH}/spack/share/spack/setup-env.sh ]; then
+        echo "Spack detected in root directory. Loading."
+        . spack/share/spack/setup-env.sh
+    else
+        echo "No spack detected. Building from source."
+        git clone --depth=100 https://github.com/spack/spack.git
+        . spack/share/spack/setup-env.sh
+    fi
 
 }
 
-install_spack_deps(){
+install_spack_deps() {
 
     # Cleaning up everything to start with a new environment
     spack uninstall -ay arch=${ARCH}
 
     spack external find [email protected]
 
-    echo "Installing libfabric..."
-    spack install [email protected] # 1.19.0 is the latest libfabric version that works on CSD3's ampere nodes
-    spack load libfabric arch=${ARCH}
-
-    echo "Installing hypre..."
-    spack install hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist \
-                  ^mpich +cuda cuda_arch=80 \
-                  ^superlu-dist +cuda cuda_arch=80 +parmetis
-    spack load hypre arch=${ARCH}
-
     echo "Installing Petsc..."
-    # spack's petsc doesn't like openmpi, but it works with mpich
+    # Spack's petsc doesn't like openmpi, but it works with mpich
     spack install petsc +cuda cuda_arch=80 +fortran +hdf5 +hypre +metis +mpi \
-              ^mpich +cuda cuda_arch=80 \
-              ^hdf5 +cxx +fortran +hl +mpi +shared
+        ^mpich +cuda cuda_arch=80 \
+        ^hdf5 +cxx +fortran +hl +mpi +shared \
+        ^hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist +cublas +gpu-aware-mpi \
+        ^superlu-dist +cuda cuda_arch=80 +parmetis +shared
     spack load petsc arch=${ARCH}
 
     echo "Installing SLEPc..."
@@ -101,16 +107,16 @@ install_spack_deps(){
 
 }
 
-install_gslib(){
+install_gslib() {
 
     echo "Installing gslib..."
-    cd ${BUILD_PATH}
+    cd "${BUILD_PATH}" || exit 1
     git clone https://github.com/Nek5000/gslib.git
-    cd gslib
+    cd gslib || exit 1
     make CC=mpicc CFLAGS="-O2 -fPIC" -j"$compile_cores"
 }
 
-install_mfem(){
+install_mfem() {
 
     export CXX=mpic++
     export CC=mpicc
@@ -119,28 +125,29 @@ install_mfem(){
     export FC=mpif90
 
     # Build MFEM
-    cd ${BUILD_PATH}
-    git clone https://github.com/Heinrich-BR/mfem.git
-    cd mfem
-    git checkout master
+    cd "${BUILD_PATH}" || exit 1
+    git clone https://github.com/mfem/mfem.git
+    cd mfem || exit 1
+    # This is just until MFEM merges Edward's changes. Without this, GPU build crashes!
+    git checkout EdwardPalmer99/add-missing-header-to-exodus-writer-fix
     mkdir build
-    cd build
+    cd build || exit 1
     echo "Building MFEM"
     cmake .. \
         -DCMAKE_BUILD_TYPE=Release \
         -DBUILD_SHARED_LIBS=YES \
         -DMFEM_USE_OPENMP=NO \
         -DMFEM_THREAD_SAFE=YES \
         -DMFEM_ENABLE_EXAMPLES=YES \
-    	-DMFEM_ENABLE_MINIAPPS=YES \
+        -DMFEM_ENABLE_MINIAPPS=YES \
         -DMFEM_USE_MPI=YES \
         -DMFEM_USE_CUDA=YES \
         -DCUDA_ARCH=sm_80 \
         -DMFEM_USE_METIS_5=YES \
         -DMFEM_USE_SUPERLU=YES \
         -DMFEM_USE_NETCDF=YES \
         -DMFEM_USE_GSLIB=YES \
-        -DGSLIB_DIR=${BUILD_PATH}/gslib/build
+        -DGSLIB_DIR="${BUILD_PATH}/gslib/build"
 
     if [ $? -eq 2 ]; then
         echo "MFEM config failed"
@@ -157,18 +164,18 @@ install_mfem(){
     LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${BUILD_PATH}/mfem/build:${BUILD_PATH}/mfem/build/miniapps/common
 }
 
-install_moose(){
+install_moose() {
 
     # Some of the variables needed
     export MOOSE_JOBS=$compile_cores
     export LIBMESH_JOBS=$compile_cores
     export METHOD="opt"
-    export SLEPC_DIR=`spack find --format "{prefix}" slepc arch=${ARCH}`
+    export SLEPC_DIR=$(spack find --format "{prefix}" slepc arch=${ARCH})
 
 
-    cd ${BUILD_PATH}
+    cd "${BUILD_PATH}" || exit 1
     git clone https://github.com/idaholab/moose
-    cd moose
+    cd moose || exit 1
 
     echo "Building libmesh..."
     ./scripts/update_and_rebuild_libmesh.sh --with-mpi
@@ -190,22 +197,22 @@ install_moose(){
         exit 1
     fi 
 
-    cd framework
+    cd framework || exit 1
     make -j"$compile_cores"
     if [ $? -eq 2 ]; then
         echo "MOOSE framework build failed"
         exit 1
     fi 
 
-    cd ../modules
+    cd ../modules || exit 1
     make -j"$compile_cores"
     if [ $? -eq 2 ]; then
         echo "MOOSE modules build failed"
         exit 1
     fi 
 
     # This takes very long! Only run the tests if you really need to!
-    #cd ../test
+    #cd ../test || exit 1
     #make -j"$compile_cores"
     #if [ $? -eq 2 ]; then
     #    echo "MOOSE test build failed"
@@ -215,26 +222,27 @@ install_moose(){
     #./run_tests -j"$compile_cores"
 }
 
-install_platypus(){
+install_platypus() {
 
-    cd ${BUILD_PATH}
+    cd "${BUILD_PATH}" || exit 1
 
     echo "Building platypus..."
     git clone https://github.com/aurora-multiphysics/platypus.git
-    cd platypus
+    cd platypus || exit 1
     git submodule update --init --recursive
-    cd contrib/hephaestus/
+    cd contrib/hephaestus/ || exit 1
     mkdir build
-    cd build
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR=${BUILD_PATH}/mfem/build ..
+    cd build || exit 1
+    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR="${BUILD_PATH}/mfem/build" ..
     ninja
-    cd ${BUILD_PATH}/platypus
+    cd ${BUILD_PATH}/platypus || exit 1
     make -j"$compile_cores"
 
 }
 
 load_modules
 set_paths
+check_spack
 install_spack_deps
 install_gslib
 install_mfem