Merge pull request #26023 from cticenhour/mumps-arm

Restore MUMPS_ARM_STR to configure_petsc.sh for manual Apple Silicon builds and elif clang-min definition changes
idaholab · Nov 16, 2023 · 2685792 · 2685792
2 parents 3014080 + fa6c6a3
commit 2685792
Show file tree

Hide file tree

Showing 17 changed files with 109 additions and 32 deletions.
diff --git a/apptainer/mpich.def b/apptainer/mpich.def
@@ -1,17 +1,31 @@
 {#- Required jinja arguments                                                                  -#}
 {#- ARCH: The machine arch                                                                    -#}
 
+{#- MOOSE_JOBS: Number of jobs to pass to the builds                                          -#}
 {#- Optional jinja arguments                                                                  -#}
 {#- ALTERNATE_FROM: Set an alternate from (currently supported: rocky9)                       -#}
 
+{#- The minimum version of LLVM/Clang                                                         -#}
+{%- set MIN_CLANG_VERSION = '10.0.1' -%}
+
+{#- The system GCC version based on ALTERNATE_FROM clang or clang-min                         -#}
+{#- Note: harmless if not set                                                                 -#}
 {%- if ALTERNATE_FROM == "clang" %}
+{%- set CLANG_GCC_VERSION = '12' -%}
+{%- elif ALTERNATE_FROM == "clang-min" %}
+{%- set CLANG_GCC_VERSION = '10' -%}
+{%- else %}
+{%- set CLANG_GCC_VERSION = '' -%}
+{%- endif %}
+
+{%- if ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang-min" %}
 Bootstrap: docker
 {%- else %}
 BootStrap: oras
 {%- endif %}
 {%- if ALTERNATE_FROM == "rocky9" %}
 From: mooseharbor.hpc.inl.gov/moose-hpcbase/rocky9-{{ ARCH }}:9.0-0
-{%- elif ALTERNATE_FROM == "clang" %}
+{%- elif ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang-min" %}
 From: rockylinux:8
 {%- elif ALTERNATE_FROM == "cuda" %}
 From: mooseharbor.hpc.inl.gov/moose-hpcbase/rocky8-cuda-{{ ARCH }}:8.6-cuda11.4.0-0
@@ -24,8 +38,13 @@ From: harbor.hpc.inl.gov/hpcbase/mpi_03:1.1.0
     export LC_ALL=C
 
 %post
-{%- if ALTERNATE_FROM == "clang" %}
+{%- if ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang-min" %}
+{%- if ALTERNATE_FROM == "clang-min" %}
+    export MIN_CLANG_VERSION={{ MIN_CLANG_VERSION }}
+{%- endif %}
     export MOOSE_MPICH_DIR=${MOOSE_MPICH_DIR:-"/opt/mpi/mpich-4.1.2"}
+    export MOOSE_JOBS={{ MOOSE_JOBS or "1" }}
+    export CLANG_GCC_VERSION={{ CLANG_GCC_VERSION }}
 {%- else %}
     export MOOSE_MPICH_DIR=${MOOSE_MPICH_DIR:-"/opt/mpi/mpich-3.4.3"}
     export MOOSE_OPENMPI_DIR=${MOOSE_OPENMPI_DIR:-"/opt/mpi/openmpi-4.1.5"}
@@ -54,10 +73,24 @@ From: harbor.hpc.inl.gov/hpcbase/mpi_03:1.1.0
                    libtool autoconf automake cppunit-devel
 
 {%- if ALTERNATE_FROM == "clang" %}
-    # Installing Clang, and its dependencies
+
+    # Installing system Clang, and its GCC GNU dependencies
     dnf install -y clang.x86_64 clang-devel.x86_64 clang-libs.x86_64 clang-tools-extra.x86_64 \
                    git-clang-format.x86_64 python3-clang.x86_64 clang-analyzer.noarch \
-                   libomp.x86_64 gcc-toolset-12-gcc-gfortran.x86_64 lldb
+                   libomp.x86_64 lldb
+{%- elif ALTERNATE_FROM == "clang-min" %}
+
+    # Installing additional packages necessary to build Clang from source
+    dnf groupinstall -y 'Development Tools'
+    dnf install -y git
+{%- endif %}
+
+{%- if ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang-min" %}
+    dnf install -y gcc-toolset-${CLANG_GCC_VERSION}
+
+    # Set alternative gcc-toolchain paths required by Clang
+    export PATH=/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/bin:${PATH}
+    export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/lib:/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/lib64:${LD_LIBRARY_PATH}
 {%- endif %}
 
     # After epel-release
@@ -115,45 +148,75 @@ EOF
     cd git-lfs-3.2.0
     ./install.sh
 
-{%- if ALTERNATE_FROM == "clang" %}
-    # Build and Install MPICH Clang
+{%- if ALTERNATE_FROM == "clang-min" %}
+
+    # Build minimum Clang {{ MIN_CLANG_VERSION }} from source
+    cd ${TEMP_LOC}
+    git clone --depth 1 --branch llvmorg-${MIN_CLANG_VERSION} https://github.com/llvm/llvm-project
+    cd llvm-project
+    mkdir llvm-build
+    cd llvm-build
+    cmake ../llvm -G 'Unix Makefiles' \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_ENABLE_PROJECTS='clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;openmp;lldb' \
+      -DCMAKE_INSTALL_PREFIX=/opt/llvm-${MIN_CLANG_VERSION}  \
+      -DCMAKE_INSTALL_RPATH:STRING=/opt/llvm-${MIN_CLANG_VERSION}/lib \
+      -DCMAKE_INSTALL_NAME_DIR:STRING=/opt/llvm-${MIN_CLANG_VERSION}/lib \
+      -DCMAKE_BUILD_WITH_INSTALL_RPATH=1 \
+      -DGCC_INSTALL_PREFIX=/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root \
+      -DCMAKE_C_COMPILER=/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/bin/gcc \
+      -DCMAKE_CXX_COMPILER=/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/bin/g++ \
+      -DCMAKE_CXX_LINK_FLAGS="-L/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/lib -Wl,-rpath,/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/lib"
+    make -j ${MOOSE_JOBS}
+    make install -j ${MOOSE_JOBS}
+
+    # Use Clang {{ MIN_CLANG_VERSION }}
+    export PATH=/opt/llvm-${MIN_CLANG_VERSION}/bin:${PATH}
+    export LD_LIBRARY_PATH=/opt/llvm-${MIN_CLANG_VERSION}/lib:${LD_LIBRARY_PATH}
+{%- endif %}
+
+{%- if ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang-min" %}
+
+    # Build and Install MPICH using Clang
     mkdir ${TEMP_LOC}/mpich
     cd ${TEMP_LOC}/mpich
     curl -L -O http://www.mpich.org/static/downloads/4.1.2/mpich-4.1.2.tar.gz
     tar -xf mpich-4.1.2.tar.gz
     mkdir mpich-4.1.2/build
     cd mpich-4.1.2/build
-    export PATH=/opt/rh/gcc-toolset-12/root/bin:${PATH}
-    export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/lib:/opt/rh/gcc-toolset-12/root/lib64:${LD_LIBRARY_PATH}
+    export CC=clang CXX=clang++ FC=gfortran F90=gfortran F77=gfortran
     ../configure --prefix=${MOOSE_MPICH_DIR} \
       --enable-shared \
-      --enable-sharedlibs=clang \
       --enable-fast=O2 \
       --enable-debuginfo \
-      --enable-totalview \
       --enable-two-level-namespace \
       CC=clang \
       CXX=clang++ \
       FC=gfortran \
       F77=gfortran \
-      F90='' \
+      F90=gfortran \
       CFLAGS='' \
       CXXFLAGS='' \
       FFLAGS='-fallow-argument-mismatch' \
       FCFLAGS='-fallow-argument-mismatch' \
       F90FLAGS='' \
       F77FLAGS=''
-    make -j ${MOOSE_JOBS:-6}
-    make install
+    make -j ${MOOSE_JOBS}
+    make install -j ${MOOSE_JOBS}
 {%- endif %}
 
     # Basic script for setting mpich environment
     USE_MPI_SCRIPT='export LD_LIBRARY_PATH=${MOOSE_MPI_DIR}/lib:${LD_LIBRARY_PATH}
 export MANPATH=${MOOSE_MPI_DIR}/share/man:${MANPATH}
 export PATH=${MOOSE_MPI_DIR}/bin:${PATH}
-{%- if ALTERNATE_FROM == "clang" %}
-export PATH=/opt/rh/gcc-toolset-12/root/bin:${PATH}
-export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-12/root/lib:/opt/rh/gcc-toolset-12/root/lib64:${LD_LIBRARY_PATH}
+{%- if ALTERNATE_FROM == "clang" or ALTERNATE_FROM == "clang-min" %}
+CLANG_GCC_VERSION={{ CLANG_GCC_VERSION }}
+export PATH=/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/bin:${PATH}
+export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/lib:/opt/rh/gcc-toolset-${CLANG_GCC_VERSION}/root/lib64:${LD_LIBRARY_PATH}
+{%- endif %}
+{%- if ALTERNATE_FROM == "clang-min" %}
+export PATH=/opt/llvm-{{ MIN_CLANG_VERSION }}/bin:${PATH}
+export LD_LIBRARY_PATH=/opt/llvm-{{ MIN_CLANG_VERSION }}/lib:${LD_LIBRARY_PATH}
 {%- endif %}
 export CC=mpicc CXX=mpicxx FC=mpif90 F90=mpif90 F77=mpif77'
 

diff --git a/conda/libmesh-vtk/conda_build_config.yaml b/conda/libmesh-vtk/conda_build_config.yaml
@@ -1,5 +1,5 @@
 moose_mpich:
-  - moose-mpich 4.0.2 build_13
+  - moose-mpich 4.0.2 build_14
 
 #### Darwin SDK SYSROOT
 CONDA_BUILD_SYSROOT:                                        # [osx]

diff --git a/conda/libmesh-vtk/meta.yaml b/conda/libmesh-vtk/meta.yaml
@@ -3,7 +3,7 @@
 #   libmesh/*
 #
 # As well as any directions pertaining to modifying those files.
-{% set build = 6 %}
+{% set build = 7 %}
 {% set vtk_version = "9.2.6" %}
 {% set vtk_friendly_version = "9.2" %}
 {% set sha256 = "06fc8d49c4e56f498c40fcb38a563ed8d4ec31358d0101e8988f0bb4d539dd12" %}

diff --git a/conda/libmesh/conda_build_config.yaml b/conda/libmesh/conda_build_config.yaml
@@ -1,8 +1,8 @@
 moose_petsc:
-  - moose-petsc 3.20.0 build_1
+  - moose-petsc 3.20.0 build_2
 
 moose_libmesh_vtk:
-  - moose-libmesh-vtk 9.2.6 build_6
+  - moose-libmesh-vtk 9.2.6 build_7
 
 #### Darwin SDK SYSROOT
 CONDA_BUILD_SYSROOT:                                        # [osx]

diff --git a/conda/libmesh/meta.yaml b/conda/libmesh/meta.yaml
@@ -4,7 +4,7 @@
 #
 # As well as any directions pertaining to modifying those files.
 # ALSO: Follow the directions in scripts/tests/versioner_hashes.yaml
-{% set build = 0 %}
+{% set build = 1 %}
 {% set version = "2023.11.06" %}
 {% set vtk_friendly_version = "9.2" %}
 

diff --git a/conda/moose-dev/conda_build_config.yaml b/conda/moose-dev/conda_build_config.yaml
@@ -1,5 +1,5 @@
 moose_libmesh:
-  - moose-libmesh 2023.11.06 build_0
+  - moose-libmesh 2023.11.06 build_1
 
 moose_wasp:
   - moose-wasp 2023.11.06
@@ -8,4 +8,4 @@ moose_tools:
   - moose-tools 2023.10.27
 
 moose_peacock:
-  - moose-peacock 2023.11.06
+  - moose-peacock 2023.11.10
diff --git a/conda/moose-dev/meta.yaml b/conda/moose-dev/meta.yaml
@@ -7,7 +7,7 @@
 #   framework/doc/packages_config.yml
 #
 # As well as any directions pertaining to modifying those files.
-{% set version = "2023.11.06" %}
+{% set version = "2023.11.10" %}
 
 package:
   name: moose-dev

diff --git a/conda/moose/conda_build_config.yaml b/conda/moose/conda_build_config.yaml
@@ -1,5 +1,5 @@
 moose_dev:
-  - moose-dev 2023.11.06
+  - moose-dev 2023.11.10
 
 #### Darwin SDK SYSROOT
 CONDA_BUILD_SYSROOT:                                        # [osx]

diff --git a/conda/mpich/meta.yaml b/conda/mpich/meta.yaml
@@ -6,7 +6,7 @@
 #
 # As well as any directions pertaining to modifying those files.
 # ALSO: Follow the directions in scripts/tests/versioner_hashes.yaml
-{% set build = 13 %}
+{% set build = 14 %}
 {% set version = "4.0.2" %}
 
 # permanent

diff --git a/conda/peacock/conda_build_config.yaml b/conda/peacock/conda_build_config.yaml
@@ -1,5 +1,5 @@
 moose_mpich:
-  - moose-mpich 4.0.2 build_13
+  - moose-mpich 4.0.2 build_14
 
 # Pesky packages that break internal CI
 # Note: Modifying/Updating this will require changes to conda/mpich!

diff --git a/conda/peacock/meta.yaml b/conda/peacock/meta.yaml
@@ -3,7 +3,7 @@
 #   moose-dev/*
 #
 # As well as any directions pertaining to modifying those files.
-{% set version = "2023.11.06" %}
+{% set version = "2023.11.10" %}
 
 package:
   name: moose-peacock

diff --git a/conda/petsc/conda_build_config.yaml b/conda/petsc/conda_build_config.yaml
@@ -1,5 +1,5 @@
 moose_mpich:
-  - moose-mpich 4.0.2 build_13
+  - moose-mpich 4.0.2 build_14
 
 #### Darwin SDK SYSROOT
 CONDA_BUILD_SYSROOT:                                        # [osx]

diff --git a/conda/petsc/meta.yaml b/conda/petsc/meta.yaml
@@ -7,7 +7,7 @@
 #
 # As well as any directions pertaining to modifying those files.
 # ALSO: Follow the directions in scripts/tests/versioner_hashes.yaml
-{% set build = 1 %}
+{% set build = 2 %}
 {% set version = "3.20.0" %}
 
 # permanent

diff --git a/conda/template/conda_build_config.yaml b/conda/template/conda_build_config.yaml
@@ -1,5 +1,5 @@
 moose_dev:
-  - moose-dev 2023.11.06
+  - moose-dev 2023.11.10
 
 #### Darwin SDK SYSROOT
 CONDA_BUILD_SYSROOT:                                        # [osx]

diff --git a/framework/doc/packages_config.yml b/framework/doc/packages_config.yml
@@ -6,4 +6,4 @@ minimum_python: 3.7
 mpich: 4.0.2
 petsc_alt: 3.11.4
 petsc: 3.20.0
-moose_dev: 2023.11.06
+moose_dev: 2023.11.10
diff --git a/scripts/configure_petsc.sh b/scripts/configure_petsc.sh
@@ -78,6 +78,13 @@ function configure_petsc()
     fi
   fi
 
+  # When manually building PETSc on Apple Silicon, set FFLAGS to the proper arch, otherwise MUMPS
+  # will fail to find MPI libraries
+  MUMPS_ARM_STR=''
+  if [[ `uname -p` == 'arm' ]] && [[ $(uname) == 'Darwin' ]] && [[ $PETSC_ARCH == 'arch-moose' ]]; then
+    MUMPS_ARM_STR='FFLAGS=-march=armv8.3-a'
+  fi
+
   cd $PETSC_DIR
   python3 ./configure --with-64-bit-indices \
       --with-cxx-dialect=C++17 \
@@ -98,6 +105,7 @@ function configure_petsc()
       --download-strumpack=1 \
       --download-superlu_dist=1 \
       $HDF5_STR \
+      $MUMPS_ARM_STR \
       $MAKE_NP_STR \
       "$@"
 

diff --git a/scripts/tests/versioner_hashes.yaml b/scripts/tests/versioner_hashes.yaml
@@ -159,3 +159,9 @@ d5e3e09267e1f2b47684a089150bec587178e1cf: #25784
   libmesh: 9ed7d66
   moose: c223a73
   wasp: 3740200
+cc86332aae2bed3d2a7574c697dfaa3540f75edb: #26023 #25992
+  mpich: a3cf7c4
+  petsc: 6d6b91e
+  libmesh: 5b1363b
+  moose: 2e50f83
+  wasp: 3740200