diff --git a/.cherry-pick b/.cherry-pick index 8484c07..f697f1e 100644 --- a/.cherry-pick +++ b/.cherry-pick @@ -1,10 +1,5 @@ -# root: fix for +opengl, remove when https://github.com/spack/spack/pull/45057 is merged -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/4505742844.diff | patch -p1 +# py-onnx: add a patch for the standard, remove after 1.15.0 +git cherry-pick bbd205543b09139f2f57f21099eb366c9f183d76 -X theirs --no-commit -# protobuf: updated patch path, remove when https://github.com/spack/spack/pull/44210 -# and https://github.com/spack/spack/pull/44443 are merged -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44210.diff | patch -p1 -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44443.diff | patch -p1 - -# dd4hep: update to version 1.29, remove when https://github.com/spack/spack/pull/44652 is merged -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44652.diff | patch -p1 +# add gaudi 39 +git cherry-pick 5ebf45861f688337dddcdee2242d4f479a832d06 -X theirs --no-commit \ No newline at end of file diff --git a/.github/workflows/concretize-template.yaml b/.github/workflows/concretize-template.yaml index 6ca3866..57feb02 100644 --- a/.github/workflows/concretize-template.yaml +++ b/.github/workflows/concretize-template.yaml @@ -13,19 +13,19 @@ jobs: fail-fast: false steps: - name: Checkout Spack - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: spack/spack path: spack - name: Checkout Key4hep - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: key4hep/key4hep-spack path: key4hep-spack - name: Checkout this repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: mucoll-spack @@ -42,7 +42,7 @@ jobs: spack spec -Nt > ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log - name: Store artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: spec-list - path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log + path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log \ No newline at end of file diff --git a/.github/workflows/concretize.yaml b/.github/workflows/concretize.yaml index d52b424..1e3fb1f 100644 --- a/.github/workflows/concretize.yaml +++ b/.github/workflows/concretize.yaml @@ -1,6 +1,6 @@ name: concretize-ubuntu -on: [push, pull_request] +on: [pull_request] jobs: concretize: @@ -10,5 +10,4 @@ jobs: target: [common, release, release-debug] uses: ./.github/workflows/concretize-template.yaml with: - target: ${{ matrix.target }} - + target: ${{ matrix.target }} \ No newline at end of file diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml new file mode 100644 index 0000000..8fcc226 --- /dev/null +++ b/.github/workflows/full-rebuild.yml @@ -0,0 +1,169 @@ +name: Re-build and publish All +on: + push: + paths: + - .github/workflows/full-rebuild.yml + - 'AlmaLinux9/Dockerfile-spack' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + id-token: write + attestations: write + +jobs: + build-spack: + runs-on: ubuntu-latest + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-spack + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker buildx + uses: docker/setup-buildx-action@v3 + - name: Login to container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Make owner name lowercase + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-spack + push: true + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + labels: ${{ steps.meta.outputs.labels }} + build-minimal: + runs-on: ubuntu-latest + needs: build-spack + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-minimal + steps: + - name: Free Disk Space (Ubuntu) + uses: endersonmenezes/free-disk-space@v2 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-minimal + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + labels: ${{ steps.meta.outputs.labels }} + build-sim: + runs-on: ubuntu-latest + needs: build-minimal + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + steps: + - name: Free Disk Space (Ubuntu) + uses: endersonmenezes/free-disk-space@v2 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-sim + build-args: | + VERSION=${{ steps.meta.outputs.version }} + IMAGE=${{ env.REGISTRY }}/${{env.OWNER_LC}} + REPOSITORY=${{ github.repository }} + MUCOLL_SHA=${{ github.sha }} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} \ No newline at end of file diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml new file mode 100644 index 0000000..3d81372 --- /dev/null +++ b/.github/workflows/minimal-rebuild.yml @@ -0,0 +1,127 @@ +name: Re-build and publish Minimal + +on: + push: + paths: + - '.github/workflows/minimal-rebuild.yml' + - 'AlmaLinux9/Dockerfile-minimal' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + id-token: write + attestations: write + +jobs: + build-minimal: + runs-on: ubuntu-latest + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-minimal + steps: + - name: Free Disk Space (Ubuntu) + uses: endersonmenezes/free-disk-space@v2 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-minimal + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + labels: ${{ steps.meta.outputs.labels }} + build-sim: + runs-on: ubuntu-latest + needs: build-minimal + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + steps: + - name: Free Disk Space (Ubuntu) + uses: endersonmenezes/free-disk-space@v2 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-sim + build-args: | + VERSION=${{ steps.meta.outputs.version }} + IMAGE=${{ env.REGISTRY }}/${{env.OWNER_LC}} + REPOSITORY=${{ github.repository }} + MUCOLL_SHA=${{ github.sha }} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} \ No newline at end of file diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml new file mode 100644 index 0000000..bf0a0c2 --- /dev/null +++ b/.github/workflows/mucoll-rebuild.yml @@ -0,0 +1,77 @@ +name: Re-build and publish MuColl + +on: + push: + paths: + - '.github/workflows/mucoll-rebuild.yml' + - '.cherry-pick' + - 'AlmaLinux9/Dockerfile-sim' + - 'environments/**' + - 'packages/**' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + id-token: write + attestations: write + +jobs: + build-sim: + runs-on: ubuntu-latest + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-sim + steps: + - name: Free Disk Space (Ubuntu) + uses: endersonmenezes/free-disk-space@v2 + with: + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-sim + build-args: | + VERSION=${{ steps.meta.outputs.version }} + IMAGE=${{ env.REGISTRY }}/${{env.OWNER_LC}} + REPOSITORY=${{ github.repository }} + MUCOLL_SHA=${{ github.sha }} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} diff --git a/.latest-commit b/.latest-commit index 649b85f..5442983 100644 --- a/.latest-commit +++ b/.latest-commit @@ -1 +1 @@ -f5946c4621035dd466953c8d2664ff5f82f38138 \ No newline at end of file +6cb16c39ab85fbc211e50be804fa7a15f24ccebc \ No newline at end of file diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal new file mode 100644 index 0000000..0512736 --- /dev/null +++ b/AlmaLinux9/Dockerfile-minimal @@ -0,0 +1,59 @@ +############################################################################### +# Repository: ${REPOSITORY}/key4hep-base +# Tag: ${VERSION}-alma9 +############################################################################### + +ARG VERSION=main +ARG REPOSITORY=madbaron +FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 + +# Picking specific key4hep commit +ARG KEY4HEP_COMMIT=475d26f0a62283a29e87b38399b128798d2b81a8 + +RUN source /opt/setup_spack.sh && \ + REPOPATH=${SPACK_ROOT}/var/key4hep-spack && \ + git clone -b base-environment --single-branch https://github.com/key4hep/key4hep-spack ${REPOPATH} && \ + if [ -n "${KEY4HEP_COMMIT}" ]; then \ + cd ${REPOPATH}; \ + git checkout ${KEY4HEP_COMMIT}; \ + fi + +# Apply our patches to spack +COPY apply_patches.sh ./apply_patches.sh + +RUN source /opt/setup_spack.sh && \ + ./apply_patches.sh ${SPACK_ROOT}/var/key4hep-spack + +# Add the package repositories +RUN source /opt/setup_spack.sh && \ + spack repo add --scope system ${SPACK_ROOT}/var/key4hep-spack + +# Create the release environment +RUN source /opt/setup_spack.sh && \ + cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-base && \ + spack env activate . && \ + cd - && \ + echo "source /opt/setup_spack.sh" > ${HOME}/setup_env.sh && \ + echo "cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-base" >> ${HOME}/setup_env.sh && \ + echo "spack env activate ." >> ${HOME}/setup_env.sh && \ + echo "cd -" >> ${HOME}/setup_env.sh && \ + echo "spack env status" >> ${HOME}/setup_env.sh + +# Concretizing the stack reusing system packages as external +RUN source ${HOME}/setup_env.sh && \ + spack add key4hep-base-stack && \ + spack concretize --reuse + +# Installing fragments of dependency tree in separate layers for cached debugging +ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" + +RUN source ${HOME}/setup_env.sh && \ + spack spec -NIt && \ + spack install ${SPACK_INSTALL_OPTS} && \ + spack clean -a + +RUN source ${HOME}/setup_env.sh && \ + echo "source ${SPACK_ENV}/.spack-env/view/setup.sh" > /opt/setup_k4base.sh && \ + echo "alias setup_k4base=\"source /opt/setup_k4base.sh\"" >> /etc/profile.d/aliases.sh + + diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim new file mode 100644 index 0000000..a515590 --- /dev/null +++ b/AlmaLinux9/Dockerfile-sim @@ -0,0 +1,65 @@ +############################################################################### +# Repository: ${IMAGE}/mucoll-sim +# Tag: ${VERSION}-alma9 +############################################################################### + +ARG VERSION=master +ARG IMAGE=madbaron +FROM ${IMAGE}/mucoll-minimal:${VERSION}-alma9 + +ARG MUCOLL_SHA=1c4ba29cebeb3e9744626ffc0d04e6ac3f22265e +ARG REPOSITORY=madbaron/mucoll-spack + +# Adding repository: MuColl +RUN source /opt/setup_spack.sh && \ + REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ + git clone https://github.com/${REPOSITORY} ${REPOPATH} && \ + cd ${REPOPATH} && \ + git checkout ${MUCOLL_SHA} && \ + cd - + +# Apply our patches to spack +COPY apply_patches.sh ./apply_patches.sh + +RUN source /opt/setup_spack.sh && \ + ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack + +# Add the package repositories +RUN source /opt/setup_spack.sh && \ + spack repo add --scope system ${SPACK_ROOT}/var/mucoll-spack + +# Create the release environment +RUN source /opt/setup_spack.sh && \ + cd ${SPACK_ROOT}/var/mucoll-spack/environments/mucoll-release && \ + spack env activate . && \ + cd - && \ + echo "source /opt/setup_spack.sh" > ${HOME}/setup_env.sh && \ + echo "cd ${SPACK_ROOT}/var/mucoll-spack/environments/mucoll-release" >> ${HOME}/setup_env.sh && \ + echo "spack env activate ." >> ${HOME}/setup_env.sh && \ + echo "cd -" >> ${HOME}/setup_env.sh && \ + echo "spack env status" >> ${HOME}/setup_env.sh + +# Concretizing the MuColl stack reusing system packages as external +RUN source ${HOME}/setup_env.sh && \ + # spack add mucoll-stack && \ + spack concretize --reuse + +# Installing fragments of dependency tree in separate layers for cached debugging +ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" + +RUN source ${HOME}/setup_env.sh && \ + spack spec -NIt && \ + spack install ${SPACK_INSTALL_OPTS} && \ + spack clean -a + +RUN source ${HOME}/setup_env.sh && \ + echo "source ${SPACK_ENV}/.spack-env/view/setup.sh" > /opt/setup_mucoll.sh && \ + echo "alias setup_mucoll=\"source /opt/setup_mucoll.sh\"" >> /etc/profile.d/aliases.sh + +# Setting up users to be used for the local environment setup +RUN useradd --shell /bin/bash --create-home mucoll +USER mucoll +WORKDIR /home/mucoll + +ENTRYPOINT ["/bin/bash", "--login"] + diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack new file mode 100644 index 0000000..096d1b5 --- /dev/null +++ b/AlmaLinux9/Dockerfile-spack @@ -0,0 +1,44 @@ +############################################################################### +# Repository: mucoll-spack +# Tag: ${VERSION}-alma9 +############################################################################### + +ARG VERSION=release +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base + +# epel-release and parallel needed for the validation +# wget needed for some tests +RUN dnf update -y && \ + dnf groupinstall -y "Development Tools" && \ + dnf install -y vim gfortran wget python3-pip epel-release mesa-libGL mesa-libGL-devel mesa-libGLU mesa-libGLU-devel krb5-devel && \ + dnf install -y parallel ccache mold && \ + dnf clean all + +RUN pip install boto3 requests + +# Add the new Certificate Authority (needed for xrootd) +RUN wget https://uit.stanford.edu/sites/default/files/2023/10/11/incommon-rsa-ca2.pem -P /usr/share/pki/ca-trust-source/anchors && \ + /usr/bin/update-ca-trust + +# Adding repositories: Spack +ARG SPACK_COMMIT=6cb16c39ab85fbc211e50be804fa7a15f24ccebc + +# Setting up Spack +RUN git clone https://github.com/spack/spack.git /opt/spack && \ + echo "source /opt/spack/share/spack/setup-env.sh" >> /opt/setup_spack.sh && \ + echo "alias setup_spack=\"source /opt/setup_spack.sh\"" >> /etc/profile.d/aliases.sh + +# Using specific commit of Key4hep repository if requested +RUN if [ -n "${SPACK_COMMIT}" ]; then \ + cd /opt/spack; \ + git checkout ${SPACK_COMMIT}; \ + fi + +#RUN source /opt/setup_spack.sh && \ +# spack install gcc@14.2.0 && \ +# spack clean -a + +# Registering compilers with Spack +ENV SPACK_COLOR="always" +RUN source /opt/setup_spack.sh && \ + spack compiler find diff --git a/AlmaLinux9/apply_patches.sh b/AlmaLinux9/apply_patches.sh new file mode 100755 index 0000000..9ba4a45 --- /dev/null +++ b/AlmaLinux9/apply_patches.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Get file to patch +if [ ${#} != 1 ]; then + echo "usage: ${0} /path/to/repo" + exit 1 +fi + +REPO=${1} + +# Determine what commit of spack we have +cd ${SPACK_ROOT} +SPACK_COMMIT=$(git rev-parse HEAD) + +# Determine key4hep supported spack commit +SPACK_COMMIT_REPO=$(cat ${REPO}/.latest-commit) + +if [ "${SPACK_COMMIT}" != "${SPACK_COMMIT_REPO}" ]; then + echo "Spack version not officially tested." + echo " recommended version: ${SPACK_COMMIT_REPO}" + echo " our version: ${SPACK_COMMIT}" + echo "Ignoring patches..." + exit 0 +fi + +# Apply the patches to spack +echo "Applying patches from ${REPO}..." +cd ${SPACK_ROOT} +source ${REPO}/.cherry-pick diff --git a/environments/mucoll-common/packages.yaml b/environments/mucoll-common/packages.yaml index 4987edd..0f1cae9 100644 --- a/environments/mucoll-common/packages.yaml +++ b/environments/mucoll-common/packages.yaml @@ -1,7 +1,7 @@ # Very loose requirements for nightly builds packages: geant4: - variants: ~qt+opengl+vecgeom cxxstd=17 + variants: ~qt+opengl+vecgeom cxxstd=20 root: variants: +davix+fftw+gsl+math+minuit+mlp+opengl~postgres~pythia6+pythia8+python+r+root7+roofit+rpath~shadow+sqlite+ssl+tbb+threads+tmva+unuran+vc+vdt+x+xml+xrootd cxxstd=17 build_type=RelWithDebInfo dd4hep: @@ -14,7 +14,7 @@ packages: variants: ~delphes_hepmc all: target: [x86_64] - variants: build_type=Release cxxstd=17 + variants: build_type=Release cxxstd=20 buildable: true providers: gl: [mesa] diff --git a/environments/mucoll-release-debug/spack.yaml b/environments/mucoll-release-debug/spack.yaml index b599c9e..219d4f1 100644 --- a/environments/mucoll-release-debug/spack.yaml +++ b/environments/mucoll-release-debug/spack.yaml @@ -1,29 +1,20 @@ spack: packages: - geant4: - require: '@11.2.0 +opengl+vecgeom' clhep: require: '@2.4.7.1 ~cms~ipo' - root: - require: '@6.32.00 +fftw+http+mlp+pythia8+r+root7+sqlite+ssl+tmva+vc' - dd4hep: - require: '@1.29 +hepmc3+lcio+xercesc' - edm4hep: - require: '@0.10.5 ~ipo' - podio: - require: '@0.17.3 ~ipo~rntuple~sio' acts: require: '@32.1.0' - whizard: - require: '@3.1.4 +lcio+openloops+openmp+pythia8' - pythia8: - require: '@8.311 ~hepmc' k4simdelphes: require: '@00-06-02 ~delphes_hepmc' - gaudi: - require: '@38.2 +aida+gaudialg' mucoll-stack: - require: '@2.9' + require: '@2.9 +devtools+pytools+ml' + + # The version seems to be necessary, otherwise it defaults to an older version + root: + require: '@6.32: +davix+fftw+gsl+math+minuit+mlp+opengl~postgres~pythia8+python+r+roofit+root7+rpath~shadow+spectrum+sqlite+ssl+tbb+threads+tmva+tmva-cpu+unuran+vc+vdt+x+xml+xrootd cxxstd=20' + + geant4: + require: +opengl+qt~vecgeom cxxstd=20 overlay: require: '@0.25' @@ -42,20 +33,54 @@ spack: muoncvxddigitiser: require: '@0.2' - gear: - require: '@1.9.2' ilcutil: require: '@1.7.2' - marlin: - require: '@1.19.1' - pandorapfa: - require: '@4.8.1' - gdb: - require: '@14.1' + + # Use the system OpenGL as recommended by spack + # See https://github.com/key4hep/key4hep-spack/issues/318 and + # https://github.com/key4hep/key4hep-spack/issues/516 + # A version is needed and we set it to 99 so that spack thinks it's the latest + # (it may not work if the versions in the current system are not what's required + # but that hasn't happened yet) + opengl: + buildable: False + externals: + - spec: opengl@99 + prefix: /usr + openglu: + buildable: False + externals: + - spec: openglu@99 + prefix: /usr + mesa: + buildable: False + externals: + - spec: mesa@99 + prefix: /usr + mesa-glu: + buildable: False + externals: + - spec: mesa-glu@99 + prefix: /usr + # Use the system kerberos, otherwise the tickets obtained at login with the + # system version don't appear for the spack kerberos (happening on lxplus at least) + krb5: + buildable: False + externals: + - spec: krb5@99 + prefix: /usr + # Use the system ssh, otherwise the configuration for the provided ssh will likely be + # different and things like ssh using kerberos won't work (by extension authentication + # for git with ssh with kerberos) + openssh: + buildable: False + externals: + - spec: openssh@99 + prefix: /usr all: target: [x86_64] - variants: build_type=Debug cxxstd=17 + variants: build_type=Debug cxxstd=20 buildable: true providers: gl: [mesa] diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index ba04917..ee02501 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -1,27 +1,25 @@ spack: + # include: + # - ../mucoll-common/compilers.yaml + packages: - geant4: - require: '@11.2.0 +opengl+vecgeom' - clhep: - require: '@2.4.7.1 ~cms~ipo' - root: - require: '+fftw+http+mlp+pythia8+r+root7+sqlite+ssl+tmva+vc+xrootd' - dd4hep: - require: '@1.29 +hepmc3+lcio+xercesc' - edm4hep: - require: '@0.10.5 ~ipo' - podio: - require: '@0.17.3 ~ipo~rntuple~sio' acts: require: '@32.1.0' - pythia8: - require: '@8.311 ~hepmc' - k4simdelphes: - require: '@00-06-02 ~delphes_hepmc' - gaudi: - require: '@38.2 +aida+gaudialg' + mucoll-stack: - require: '@2.9 +devtools' + require: '@master +devtools+pytools+ml' + + py-setuptools: + require: "@69" + + # The version seems to be necessary, otherwise it defaults to an older version + root: + require: '@6.32: +davix+fftw+gsl+math+minuit+mlp+opengl~postgres~pythia8+python+r+roofit+root7+rpath~shadow+spectrum+sqlite+ssl+tbb+threads+tmva+tmva-cpu+unuran+vc+vdt+x+xml+xrootd cxxstd=20' + + geant4: + require: +opengl+qt~vecgeom cxxstd=20 + dd4hep: + require: +edm4hep+lcio+xercesc+hepmc3 overlay: require: '@0.25' @@ -34,28 +32,64 @@ spack: conformaltracking: require: '@1.12.1' marlintrkprocessors: - require: '@2.15.2' + require: '@master' actstracking: - require: '@1.2.2' + require: '@1.3.0' muoncvxddigitiser: - require: '@0.2' - - gear: - require: '@1.9.2' - ilcutil: - require: '@1.7.2' - marlin: - require: '@1.19.1' - pandorapfa: - require: '@4.8.1' - gdb: - require: '@14.1' + require: '@master' + + # Use the system OpenGL as recommended by spack + # See https://github.com/key4hep/key4hep-spack/issues/318 and + # https://github.com/key4hep/key4hep-spack/issues/516 + # A version is needed and we set it to 99 so that spack thinks it's the latest + # (it may not work if the versions in the current system are not what's required + # but that hasn't happened yet) + opengl: + buildable: False + externals: + - spec: opengl@99 + prefix: /usr + openglu: + buildable: False + externals: + - spec: openglu@99 + prefix: /usr + mesa: + buildable: False + externals: + - spec: mesa@99 + prefix: /usr + mesa-glu: + buildable: False + externals: + - spec: mesa-glu@99 + prefix: /usr + # Use the system kerberos, otherwise the tickets obtained at login with the + # system version don't appear for the spack kerberos (happening on lxplus at least) + krb5: + buildable: False + externals: + - spec: krb5@99 + prefix: /usr + # Use the system ssh, otherwise the configuration for the provided ssh will likely be + # different and things like ssh using kerberos won't work (by extension authentication + # for git with ssh with kerberos) + openssh: + buildable: False + externals: + - spec: openssh@99 + prefix: /usr all: target: [x86_64] - variants: build_type=Release cxxstd=17 + variants: build_type=RelWithDebInfo cxxstd=20 buildable: true providers: gl: [mesa] glu: [mesa] + concretizer: + unify: true + + specs: + - mucoll-stack \ No newline at end of file diff --git a/packages/actstracking/package.py b/packages/actstracking/package.py index d5bdb34..887cd15 100644 --- a/packages/actstracking/package.py +++ b/packages/actstracking/package.py @@ -16,7 +16,8 @@ class Actstracking(CMakePackage, MCIlcsoftpackage): maintainers = ['gianelle', 'kkrizka'] version('main', branch='main') - version('1.2.2', sha256='be08b87037167892a9b1a7ad601511beaf99423e836841436c6318fef5fa93de', preferred=True) + version('1.3.0', sha256='d013a7700ce453054848572603bcfc6fdf4f5a4d', preferred=True) + version('1.2.2', sha256='be08b87037167892a9b1a7ad601511beaf99423e836841436c6318fef5fa93de') version('1.2.1', sha256='747c15a4c937ab09d79afcc956bb1f1f82ce345febfb4bd18462b71e70ae0b29') version('1.2', sha256='7390d03ab848f7ad9e67c5aabda8122942a885256775174db30964fb9fe028e1') version('1.1.0', sha256='d565e70a2fec97d0d2e81ada69ed54ef8dacc44b0f608b4cf3dffa561091afeb') @@ -39,6 +40,10 @@ def setup_run_environment(self, spack_env): spack_env.prepend_path('MARLIN_DLL', self.prefix.lib + "/libACTSTracking.so") spack_env.set("ACTS_TGeoFile", self.prefix.share.ACTSTracking.data + "/MuColl_v1.root") spack_env.set("ACTS_MatFile", self.prefix.share.ACTSTracking.data + "/material-maps.json") + spack_env.set("ACTS_TGeoFile_MuSIC", self.prefix.share.ACTSTracking.data + "/MuSIC_v2.root") + spack_env.set("ACTS_MatFile_MuSIC", self.prefix.share.ACTSTracking.data + "/material-maps.json") + spack_env.set("ACTS_TGeoFile_MAIA", self.prefix.share.ACTSTracking.data + "/MAIA_v0.root") + spack_env.set("ACTS_MatFile_MAIA", self.prefix.share.ACTSTracking.data + "/MAIA_v0_material.json") def cmake_args(self): # C++ Standard diff --git a/packages/gaudi/package.py b/packages/gaudi/package.py deleted file mode 100644 index f8d906a..0000000 --- a/packages/gaudi/package.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.package import * -from spack.pkg.builtin.boost import Boost - - -class Gaudi(CMakePackage): - """An experiment-independent HEP event data processing framework""" - - homepage = "https://gaudi.web.cern.ch/gaudi/" - git = "https://gitlab.cern.ch/gaudi/Gaudi.git" - url = "https://gitlab.cern.ch/gaudi/Gaudi/-/archive/v33r1/Gaudi-v33r1.tar.gz" - - tags = ["hep"] - - version("master", branch="master") - version("38.2", sha256="08759b1398336987ad991602e37079f0744e8d8e4e3d5df2d253b8dedf925068") - version("38.1", sha256="79d42833edcebc2099f91badb6f72708640c05f678cc4521a86e857f112486dc") - version("38.0", sha256="52f2733fa0af760c079b3438bb9c7e36b28ea704f78b0085458e1918c11e1653") - version("37.2", sha256="9b866caab46e182de98b59eddbde80d6fa0e670fe4a35906f1518b04bd99b2d2") - version("37.1", sha256="1d7038fd5dfb5f2517ce57623cf8090549ffe2ea8f0171d534e5c1ca20bd009a") - version("37.0", sha256="823f3821a4f498ddd2dd123fbb8a3787b361ddfd818f4ab13572076fc9afdfe4") - version("36.14", sha256="b11e0afcb797d61a305856dfe8079d48d74c6b6867ceccc0a83aab5978c9ba5f") - version("36.13", sha256="41e711c83428663996c825044b268ce515bef85dad74b4a9453f2207b4b1be7b") - version("36.12", sha256="dfce9156cedfa0a7234f880a3c395e592a5f3dc79070d5d196fdb94b83ae203e") - version("36.11", sha256="81664d033b0aa8598a0e4cb7e455e697baeb063a11bbde2390164776238ba9f7") - version("36.10", sha256="2c1f181c54a76b493b913aeecbd6595236afc08e41d7f1d80be6fe65ac95adb3") - version("36.9", sha256="b4e080094771f111bd0bcdf744bcab7b028c7e2af7c5dfaa4a977ebbf0160a8f") - version("36.8", sha256="64b4300a57335af7c1f74c736d7610041a1ef0c1f976e3342a22385b60519afc") - version("36.7", sha256="8dca43185ba11e1b33f5535d2e384542d84500407b0d1f8cb920be00f05c9716") - version("36.6", sha256="8fc7be0ce32f99cc6b0be4ebbb246f4bb5008ffbf0c012cb39c0aff813dce6af") - version("36.5", sha256="593e0316118411a5c5fde5d4d87cbfc3d2bb748a8c72a66f4025498fcbdb0f7e") - version("36.4", sha256="1a5c27cdc21ec136b47f5805406c92268163393c821107a24dbb47bd88e4b97d") - version("36.3", sha256="9ac228d8609416afe4dea6445c6b3ccebac6fab1e46121fcc3a056e24a5d6640") - version("36.2", sha256="a1b4bb597941a7a5b8d60382674f0b4ca5349c540471cd3d4454efbe7b9a09b9") - version("36.1", sha256="9f718c832313676249e5c3ac76ba4346978ee2328f8cdcb29176498b080402e9") - version("36.0", sha256="8a0458cef5b616532f9db7cca9fa0e892e602b64c9e93dc0cc6d972e03034830") - version("35.0", sha256="c01b822f9592a7bf875b9997cbeb3c94dea97cb13d523c12649dbbf5d69b5fa6") - - maintainers("drbenmorgan", "vvolkl", "jmcarcell") - - variant("aida", default=False, description="Build AIDA interfaces support") - variant("cppunit", default=False, description="Build with CppUnit unit testing") - variant("docs", default=False, description="Build documentation with Doxygen") - variant("examples", default=False, description="Build examples") - variant("gaudialg", default=False, description="Build GaudiAlg support", when="@37.0:") - variant("gperftools", default=False, description="Build with Google PerfTools support") - variant("heppdt", default=False, description="Build with HEP Particle Data Table support") - variant("jemalloc", default=False, description="Build with jemalloc allocator support") - variant("unwind", default=False, description="Build with unwind call-chains") - variant("vtune", default=False, description="Build with Intel VTune profiler support") - variant("xercesc", default=False, description="Build with Xerces-C XML support") - - # These dependencies are needed for a minimal Gaudi build - depends_on("aida") - depends_on("boost@1.67.0: +python") - - # TODO: replace this with an explicit list of components of Boost, - # for instance depends_on('boost +filesystem') - # See https://github.com/spack/spack/pull/22303 for reference - depends_on(Boost.with_default_variants) - depends_on("clhep") - depends_on("cmake", type="build") - depends_on("cppgsl") - depends_on("fmt") - depends_on("fmt@:8", when="@:36.9") - depends_on("intel-tbb@:2020.3", when="@:37.0") - depends_on("tbb", when="@37.1:") - depends_on("uuid") - depends_on("nlohmann-json") - depends_on("python", type=("build", "run")) - depends_on("py-networkx", type=("build", "run")) - depends_on("py-six", type=("build", "run")) - depends_on("py-pyyaml", type=("build", "run", "test")) - depends_on("range-v3") - depends_on("root +python +root7 +ssl +tbb +threads") - depends_on("zlib-api") - - # Testing dependencies - # Note: gaudi only builds examples when testing enabled - for pv in (["catch2", "@36.8:"], ["py-nose", "@35:"], ["py-pytest", "@36.2:"]): - depends_on(pv[0], when=pv[1], type="test") - depends_on(pv[0], when=pv[1] + " +examples") - - # Adding these dependencies triggers the build of most optional components - depends_on("cppgsl", when="+cppunit") - depends_on("cppunit", when="+cppunit") - depends_on("doxygen +graphviz", when="+docs") - depends_on("gperftools", when="+gperftools") - depends_on("gdb") - depends_on("heppdt", when="+heppdt") - depends_on("jemalloc", when="+jemalloc") - depends_on("libunwind", when="+unwind") - depends_on("xerces-c", when="+xercesc") - # NOTE: pocl cannot be added as a minimal OpenCL implementation because - # ROOT does not like being exposed to LLVM symbols. - - # The Intel VTune dependency is taken aside because it requires a license - depends_on("intel-parallel-studio -mpi +vtune", when="+vtune") - - def cmake_args(self): - args = [ - # Note: gaudi only builds examples when testing enabled - self.define("BUILD_TESTING", self.run_tests or self.spec.satisfies("+examples")), - self.define_from_variant("GAUDI_USE_AIDA", "aida"), - self.define_from_variant("GAUDI_USE_CPPUNIT", "cppunit"), - self.define_from_variant("GAUDI_ENABLE_GAUDIALG", "gaudialg"), - self.define_from_variant("GAUDI_USE_GPERFTOOLS", "gperftools"), - self.define_from_variant("GAUDI_USE_HEPPDT", "heppdt"), - self.define_from_variant("GAUDI_USE_JEMALLOC", "jemalloc"), - self.define_from_variant("GAUDI_USE_UNWIND", "unwind"), - self.define_from_variant("GAUDI_USE_XERCESC", "xercesc"), - self.define_from_variant("GAUDI_USE_DOXYGEN", "docs"), - # needed to build core services like rndmsvc - self.define("GAUDI_USE_CLHEP", True), - self.define("GAUDI_USE_PYTHON_MAJOR", str(self.spec["python"].version.up_to(1))), - # todo: - self.define("GAUDI_USE_INTELAMPLIFIER", False), - ] - return args - - def setup_run_environment(self, env): - # environment as in Gaudi.xenv - env.prepend_path("PATH", self.prefix.scripts) - env.prepend_path("PYTHONPATH", self.prefix.python) - env.prepend_path("LD_LIBRARY_PATH", self.prefix.lib) - env.prepend_path("LD_LIBRARY_PATH", self.prefix.lib64) - - def url_for_version(self, version): - major = str(version[0]) - minor = str(version[1]) - url = "https://gitlab.cern.ch/gaudi/Gaudi/-/archive/v{0}r{1}/Gaudi-v{0}r{1}.tar.gz".format( - major, minor - ) - return url diff --git a/packages/mucoll-stack/package.py b/packages/mucoll-stack/package.py index 7103bff..1f0a77f 100644 --- a/packages/mucoll-stack/package.py +++ b/packages/mucoll-stack/package.py @@ -17,7 +17,7 @@ class MucollStack(BundlePackage, Key4hepPackage): homepage = 'https://github.com/MuonColliderSoft' - maintainers = ['bartosik-hep'] + maintainers = ['bartosik-hep', 'madbaron'] ##################### versions ######################## ####################################################### @@ -26,6 +26,8 @@ class MucollStack(BundlePackage, Key4hepPackage): # should use `environments/mucoll-common/packages.yaml` version(datetime.today().strftime('%Y-%m-%d')) + version("master", branch="master") + ### stable build # to install exact specified version for every dependecy # should use `environments/mucoll-release/packages.yaml` @@ -41,7 +43,9 @@ class MucollStack(BundlePackage, Key4hepPackage): variant('build_type', default='Release', description='CMake build type', values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel')) - + variant('llvm', default=False, description='Build with LLVM') + variant('ml', default=False, description='Build with machine learning tools') + variant('pytools', default=False, description='Build with python tools') ############################### Key4hep ############### ####################################################### @@ -108,16 +112,22 @@ class MucollStack(BundlePackage, Key4hepPackage): depends_on('ninja') depends_on('doxygen') depends_on('gdb') - depends_on('llvm') + + depends_on('llvm', when='+llvm') + + with when('+ml'): + # ML tools depends_on('onnx') depends_on('xgboost') + depends_on('py-onnxruntime') + depends_on('py-onnx') + + with when('+pytools'): # Python tools depends_on('py-h5py') depends_on('py-ipython') depends_on('py-jupytext') depends_on('py-matplotlib') - depends_on('py-onnxruntime') - depends_on('py-onnx') depends_on('py-pandas') depends_on('py-particle') depends_on('py-pip') @@ -141,13 +151,15 @@ def setup_run_environment(self, env): env.set('MUCOLL_GEO', os.path.join(self.spec['lcgeo'].prefix.share.lcgeo.compact, 'MuColl/MuColl_v1/MuColl_v1.xml')) env.set('MUCOLL_RELEASE_VERSION', self.spec.version) + # ROOT needs to be in LD_LIBRARY_PATH to prevent using system installations + env.prepend_path("LD_LIBRARY_PATH", self.spec["root"].prefix.lib) + env.prepend_path("PYTHONPATH", self.spec["root"].prefix.lib) + # set vdt, needed for root, see https://github.com/spack/spack/pull/37278 if "vdt" in self.spec: env.prepend_path("CPATH", self.spec["vdt"].prefix.include) # When building podio with +rntuple there are warnings constantly without this env.prepend_path("LD_LIBRARY_PATH", self.spec["vdt"].libs.directories[0]) - - env.prepend_path('LD_LIBRARY_PATH', self.spec['root'].prefix.lib.root) # remove when https://github.com/spack/spack/pull/37881 is merged env.prepend_path('LD_LIBRARY_PATH', self.spec['podio'].libs.directories[0])