From 3ff04277d2743a36bc4e6b6d162af22e3d2e4bd7 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 14 Nov 2024 10:55:31 +0100 Subject: [PATCH 01/80] updates for 2.10 --- environments/mucoll-release/spack.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index ba04917..29dd38c 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -46,8 +46,6 @@ spack: require: '@1.7.2' marlin: require: '@1.19.1' - pandorapfa: - require: '@4.8.1' gdb: require: '@14.1' From c3ccf9c15ecd6c56a4e80a641c4acc2a8bebc4dd Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Wed, 20 Nov 2024 21:47:09 +0100 Subject: [PATCH 02/80] update spack commit --- .latest-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.latest-commit b/.latest-commit index 649b85f..5442983 100644 --- a/.latest-commit +++ b/.latest-commit @@ -1 +1 @@ -f5946c4621035dd466953c8d2664ff5f82f38138 \ No newline at end of file +6cb16c39ab85fbc211e50be804fa7a15f24ccebc \ No newline at end of file From bb4d8644b7b71ad7db7c34e68dac9353b743aa93 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 09:40:44 +0100 Subject: [PATCH 03/80] updating cherry picks --- .cherry-pick | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.cherry-pick b/.cherry-pick index 8484c07..e69de29 100644 --- a/.cherry-pick +++ b/.cherry-pick @@ -1,10 +0,0 @@ -# root: fix for +opengl, remove when https://github.com/spack/spack/pull/45057 is merged -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/4505742844.diff | patch -p1 - -# protobuf: updated patch path, remove when https://github.com/spack/spack/pull/44210 -# and https://github.com/spack/spack/pull/44443 are merged -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44210.diff | patch -p1 -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44443.diff | patch -p1 - -# dd4hep: update to version 1.29, remove when https://github.com/spack/spack/pull/44652 is merged -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/44652.diff | patch -p1 From c77399b9f1500dd4ad80fcaa067ae44fb566503e Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 10:27:28 +0100 Subject: [PATCH 04/80] split build towards 2.10 --- environments/mucoll-release/spack.yaml | 32 +++++++++----------------- 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 29dd38c..3fbf627 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -1,28 +1,17 @@ spack: packages: - geant4: - require: '@11.2.0 +opengl+vecgeom' clhep: require: '@2.4.7.1 ~cms~ipo' - root: - require: '+fftw+http+mlp+pythia8+r+root7+sqlite+ssl+tmva+vc+xrootd' - dd4hep: - require: '@1.29 +hepmc3+lcio+xercesc' - edm4hep: - require: '@0.10.5 ~ipo' - podio: - require: '@0.17.3 ~ipo~rntuple~sio' acts: require: '@32.1.0' - pythia8: - require: '@8.311 ~hepmc' k4simdelphes: require: '@00-06-02 ~delphes_hepmc' - gaudi: - require: '@38.2 +aida+gaudialg' mucoll-stack: require: '@2.9 +devtools' + geant4: + require: +opengl+qt~vecgeom cxxstd=20 + overlay: require: '@0.25' lcgeo: @@ -40,20 +29,21 @@ spack: muoncvxddigitiser: require: '@0.2' - gear: - require: '@1.9.2' ilcutil: require: '@1.7.2' - marlin: - require: '@1.19.1' - gdb: - require: '@14.1' all: target: [x86_64] - variants: build_type=Release cxxstd=17 + variants: build_type=RelWithDebInfo cxxstd=20 buildable: true providers: gl: [mesa] glu: [mesa] + specs: + - mucoll-stack + - geant4/p7jlshk + - root/voacefc + + concretizer: + unify: true \ No newline at end of file From 28bfada61262a436a6564441edeb7bf8c826b7c0 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 10:31:41 +0100 Subject: [PATCH 05/80] specifying root version --- environments/mucoll-release/spack.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 3fbf627..1bc7681 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -9,6 +9,10 @@ spack: mucoll-stack: require: '@2.9 +devtools' + # The version seems to be necessary, otherwise it defaults to an older version + root: + require: '@6.32: +davix+fftw+gsl+math+minuit+mlp+opengl~postgres~pythia8+python+r+roofit+root7+rpath~shadow+spectrum+sqlite+ssl+tbb+threads+tmva+tmva-cpu+unuran+vc+vdt+x+xml+xrootd cxxstd=20' + geant4: require: +opengl+qt~vecgeom cxxstd=20 From 11a32828ca9230337562dc43f91a25862d3a21ff Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 11:03:23 +0100 Subject: [PATCH 06/80] got rid of hardcoded hashes --- environments/mucoll-release/spack.yaml | 52 +++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 1bc7681..018ad73 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -36,6 +36,48 @@ spack: ilcutil: require: '@1.7.2' + # Use the system OpenGL as recommended by spack + # See https://github.com/key4hep/key4hep-spack/issues/318 and + # https://github.com/key4hep/key4hep-spack/issues/516 + # A version is needed and we set it to 99 so that spack thinks it's the latest + # (it may not work if the versions in the current system are not what's required + # but that hasn't happened yet) + opengl: + buildable: False + externals: + - spec: opengl@99 + prefix: /usr + openglu: + buildable: False + externals: + - spec: openglu@99 + prefix: /usr + mesa: + buildable: False + externals: + - spec: mesa@99 + prefix: /usr + mesa-glu: + buildable: False + externals: + - spec: mesa-glu@99 + prefix: /usr + # Use the system kerberos, otherwise the tickets obtained at login with the + # system version don't appear for the spack kerberos (happening on lxplus at least) + krb5: + buildable: False + externals: + - spec: krb5@99 + prefix: /usr + # Use the system ssh, otherwise the configuration for the provided ssh will likely be + # different and things like ssh using kerberos won't work (by extension authentication + # for git with ssh with kerberos) + openssh: + buildable: False + externals: + - spec: openssh@99 + prefix: /usr + all: target: [x86_64] variants: build_type=RelWithDebInfo cxxstd=20 @@ -44,10 +86,8 @@ spack: gl: [mesa] glu: [mesa] - specs: - - mucoll-stack - - geant4/p7jlshk - - root/voacefc - concretizer: - unify: true \ No newline at end of file + unify: true + + specs: + - mucoll-stack \ No newline at end of file From 07196b06f93d50d7951e27f460cca025f3cd12af Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 14:13:35 +0100 Subject: [PATCH 07/80] integrating docker image building --- .github/workflows/full-rebuild.yml | 157 +++++++++++++++++++++++++ .github/workflows/minimal-rebuild.yml | 108 +++++++++++++++++ .github/workflows/mucoll-rebuild.yml | 65 ++++++++++ AlmaLinux9/Dockerfile-minimal | 59 ++++++++++ AlmaLinux9/Dockerfile-sim | 65 ++++++++++ AlmaLinux9/Dockerfile-spack | 33 ++++++ AlmaLinux9/apply_patches.sh | 29 +++++ environments/mucoll-release/spack.yaml | 2 +- packages/mucoll-stack/package.py | 15 ++- 9 files changed, 527 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/full-rebuild.yml create mode 100644 .github/workflows/minimal-rebuild.yml create mode 100644 .github/workflows/mucoll-rebuild.yml create mode 100644 AlmaLinux9/Dockerfile-minimal create mode 100644 AlmaLinux9/Dockerfile-sim create mode 100644 AlmaLinux9/Dockerfile-spack create mode 100755 AlmaLinux9/apply_patches.sh diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml new file mode 100644 index 0000000..f07f6a4 --- /dev/null +++ b/.github/workflows/full-rebuild.yml @@ -0,0 +1,157 @@ +name: Re-build and publish everything + +on: + push: + paths: + - 'AlmaLinux9/Dockerfile-spack' + tags: + - 'v*' + pull_request: + branches: + - 'master' + - 'main' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + id-token: write + attestations: write + +jobs: + build-spack: + runs-on: ubuntu-latest + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-spack + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker buildx + uses: docker/setup-buildx-action@v3 + - name: Login to container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Make owner name lowercase + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-spack + push: true + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + labels: ${{ steps.meta.outputs.labels }} + build-minimal: + runs-on: ubuntu-latest + needs: build-spack + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-minimal + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-minimal + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + labels: ${{ steps.meta.outputs.labels }} + build-sim: + runs-on: ubuntu-latest + needs: build-minimal + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-sim + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml new file mode 100644 index 0000000..dad256b --- /dev/null +++ b/.github/workflows/minimal-rebuild.yml @@ -0,0 +1,108 @@ +name: Re-build and publish Minimal + +on: + push: + paths: + - 'AlmaLinux9/Dockerfile-base' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + id-token: write + attestations: write + +jobs: + build-minimal: + runs-on: ubuntu-latest + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-minimal + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-minimal + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + labels: ${{ steps.meta.outputs.labels }} + build-sim: + runs-on: ubuntu-latest + needs: build-minimal + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-sim + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml new file mode 100644 index 0000000..72081dc --- /dev/null +++ b/.github/workflows/mucoll-rebuild.yml @@ -0,0 +1,65 @@ +name: Re-build and publish MuColl + +on: + push: + paths: + - 'AlmaLinux9/Dockerfile-sim' + - 'environments/**' + - 'packages/**' + workflow_dispatch: + +env: + REGISTRY: ghcr.io + +permissions: + contents: read + packages: write + id-token: write + attestations: write + +jobs: + build-sim: + runs-on: ubuntu-latest + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + env: + IMAGE_NAME: mucoll-sim + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: true + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-sim + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal new file mode 100644 index 0000000..3d2f544 --- /dev/null +++ b/AlmaLinux9/Dockerfile-minimal @@ -0,0 +1,59 @@ +############################################################################### +# Repository: ${REPOSITORY}/key4hep-base +# Tag: ${VERSION}-alma9 +############################################################################### + +ARG VERSION=main +ARG REPOSITORY=madbaron +FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 + +# Adding Key4hep repo +ARG KEY4HEP_COMMIT=9f4187ee47dfa1828be936e34c23035818bcd3aa + +RUN source /opt/setup_spack.sh && \ + REPOPATH=${SPACK_ROOT}/var/key4hep-spack && \ + git clone -b base-environment --single-branch https://github.com/key4hep/key4hep-spack ${REPOPATH} && \ + if [ -n "${KEY4HEP_COMMIT}" ]; then \ + cd ${REPOPATH}; \ + git checkout ${KEY4HEP_COMMIT}; \ + fi + +# Apply our patches to spack +COPY apply_patches.sh ./apply_patches.sh + +RUN source /opt/setup_spack.sh && \ + ./apply_patches.sh ${SPACK_ROOT}/var/key4hep-spack + +# Add the package repositories +RUN source /opt/setup_spack.sh && \ + spack repo add --scope system ${SPACK_ROOT}/var/key4hep-spack + +# Create the release environment +RUN source /opt/setup_spack.sh && \ + cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-base && \ + spack env activate . && \ + cd - && \ + echo "source /opt/setup_spack.sh" > ${HOME}/setup_env.sh && \ + echo "cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-base" >> ${HOME}/setup_env.sh && \ + echo "spack env activate ." >> ${HOME}/setup_env.sh && \ + echo "cd -" >> ${HOME}/setup_env.sh && \ + echo "spack env status" >> ${HOME}/setup_env.sh + +# Concretizing the stack reusing system packages as external +RUN source ${HOME}/setup_env.sh && \ + spack add key4hep-base-stack && \ + spack concretize --reuse + +# Installing fragments of dependency tree in separate layers for cached debugging +ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" + +RUN source ${HOME}/setup_env.sh && \ + spack spec -NIt && \ + spack install ${SPACK_INSTALL_OPTS} && \ + spack clean -a + +RUN source ${HOME}/setup_env.sh && \ + echo "source ${SPACK_ENV}/.spack-env/view/setup.sh" > /opt/setup_k4base.sh && \ + echo "alias setup_k4base=\"source /opt/setup_k4base.sh\"" >> /etc/profile.d/aliases.sh + + diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim new file mode 100644 index 0000000..37514e5 --- /dev/null +++ b/AlmaLinux9/Dockerfile-sim @@ -0,0 +1,65 @@ +############################################################################### +# Repository: ${REPOSITORY}/mucoll-sim +# Tag: ${VERSION}-alma9 +############################################################################### + +ARG VERSION=dev +ARG REPOSITORY=madbaron +FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 + +# Adding repository: MuColl +ARG MUCOLL_COMMIT=11a32828ca9230337562dc43f91a25862d3a21ff + +RUN source /opt/setup_spack.sh && \ + REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ + git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ + if [ -n "${MUCOLL_COMMIT}" ]; then \ + cd ${REPOPATH}; \ + git checkout ${MUCOLL_COMMIT}; \ + fi + +# Apply our patches to spack +COPY apply_patches.sh ./apply_patches.sh + +RUN source /opt/setup_spack.sh && \ + ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack + +# Add the package repositories +RUN source /opt/setup_spack.sh && \ + spack repo add --scope system ${SPACK_ROOT}/var/mucoll-spack + +# Create the release environment +RUN source /opt/setup_spack.sh && \ + cd ${SPACK_ROOT}/var/mucoll-spack/environments/mucoll-release && \ + spack env activate . && \ + cd - && \ + echo "source /opt/setup_spack.sh" > ${HOME}/setup_env.sh && \ + echo "cd ${SPACK_ROOT}/var/mucoll-spack/environments/mucoll-release" >> ${HOME}/setup_env.sh && \ + echo "spack env activate ." >> ${HOME}/setup_env.sh && \ + echo "cd -" >> ${HOME}/setup_env.sh && \ + echo "spack env status" >> ${HOME}/setup_env.sh + +# Concretizing the MuColl stack reusing system packages as external +RUN source ${HOME}/setup_env.sh && \ + spack add mucoll-stack && \ + spack concretize --reuse + +# Installing fragments of dependency tree in separate layers for cached debugging +ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" + +RUN source ${HOME}/setup_env.sh && \ + spack spec -NIt && \ + spack install ${SPACK_INSTALL_OPTS} && \ + spack clean -a + +RUN source ${HOME}/setup_env.sh && \ + echo "source ${SPACK_ENV}/.spack-env/view/setup.sh" > /opt/setup_mucoll.sh && \ + echo "alias setup_mucoll=\"source /opt/setup_mucoll.sh\"" >> /etc/profile.d/aliases.sh + +# Setting up users to be used for the local environment setup +RUN useradd --shell /bin/bash --create-home mucoll +USER mucoll +WORKDIR /home/mucoll + +ENTRYPOINT ["/bin/bash", "--login"] + diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack new file mode 100644 index 0000000..0509b73 --- /dev/null +++ b/AlmaLinux9/Dockerfile-spack @@ -0,0 +1,33 @@ +############################################################################### +# Repository: mucoll-spack +# Tag: ${VERSION}-alma9 +############################################################################### + +ARG VERSION=release +FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build +LABEL org.opencontainers.image.source=https://github.com/madbaron/MuonCollider-docker +LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" +LABEL org.opencontainers.image.licenses=Apache-2.0 + +# Add the new Certificate Authority (needed for xrootd) +RUN wget https://uit.stanford.edu/sites/default/files/2023/10/11/incommon-rsa-ca2.pem -P /usr/share/pki/ca-trust-source/anchors && \ + /usr/bin/update-ca-trust + +# Adding repositories: Spack +ARG SPACK_COMMIT=6cb16c39ab85fbc211e50be804fa7a15f24ccebc + +# Setting up Spack +RUN git clone https://github.com/spack/spack.git /opt/spack && \ + echo "source /opt/spack/share/spack/setup-env.sh" >> /opt/setup_spack.sh && \ + echo "alias setup_spack=\"source /opt/setup_spack.sh\"" >> /etc/profile.d/aliases.sh + +# Using specific commit of Key4hep repository if requested +RUN if [ -n "${SPACK_COMMIT}" ]; then \ + cd /opt/spack; \ + git checkout ${SPACK_COMMIT}; \ + fi + +# Registering compilers with Spack +ENV SPACK_COLOR="always" +RUN source /opt/setup_spack.sh && \ + spack compiler find diff --git a/AlmaLinux9/apply_patches.sh b/AlmaLinux9/apply_patches.sh new file mode 100755 index 0000000..9ba4a45 --- /dev/null +++ b/AlmaLinux9/apply_patches.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Get file to patch +if [ ${#} != 1 ]; then + echo "usage: ${0} /path/to/repo" + exit 1 +fi + +REPO=${1} + +# Determine what commit of spack we have +cd ${SPACK_ROOT} +SPACK_COMMIT=$(git rev-parse HEAD) + +# Determine key4hep supported spack commit +SPACK_COMMIT_REPO=$(cat ${REPO}/.latest-commit) + +if [ "${SPACK_COMMIT}" != "${SPACK_COMMIT_REPO}" ]; then + echo "Spack version not officially tested." + echo " recommended version: ${SPACK_COMMIT_REPO}" + echo " our version: ${SPACK_COMMIT}" + echo "Ignoring patches..." + exit 0 +fi + +# Apply the patches to spack +echo "Applying patches from ${REPO}..." +cd ${SPACK_ROOT} +source ${REPO}/.cherry-pick diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 018ad73..2426290 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -7,7 +7,7 @@ spack: k4simdelphes: require: '@00-06-02 ~delphes_hepmc' mucoll-stack: - require: '@2.9 +devtools' + require: '@2.9 +devtools+pytools+ml' # The version seems to be necessary, otherwise it defaults to an older version root: diff --git a/packages/mucoll-stack/package.py b/packages/mucoll-stack/package.py index 95d480a..439715e 100644 --- a/packages/mucoll-stack/package.py +++ b/packages/mucoll-stack/package.py @@ -17,7 +17,7 @@ class MucollStack(BundlePackage, Key4hepPackage): homepage = 'https://github.com/MuonColliderSoft' - maintainers = ['bartosik-hep'] + maintainers = ['bartosik-hep', 'madbaron'] ##################### versions ######################## ####################################################### @@ -41,7 +41,9 @@ class MucollStack(BundlePackage, Key4hepPackage): variant('build_type', default='Release', description='CMake build type', values=('Debug', 'Release', 'RelWithDebInfo', 'MinSizeRel')) - + variant('llvm', default=False, description='Build with LLVM') + variant('ml', default=False, description='Build with machine learning tools') + variant('pytools', default=False, description='Build with python tools') ############################### Key4hep ############### ####################################################### @@ -108,9 +110,12 @@ class MucollStack(BundlePackage, Key4hepPackage): depends_on('ninja') depends_on('doxygen') depends_on('gdb') - depends_on('llvm') - depends_on('onnx') - depends_on('xgboost') + + depends_on('llvm', when='+llvm') + depends_on('onnx', when='+ml') + depends_on('xgboost', when='+ml') + + with when('+pytools'): # Python tools depends_on('py-h5py') depends_on('py-ipython') From 4e0f004d8479fa4f53178e18071896270e10ae97 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 14:16:33 +0100 Subject: [PATCH 08/80] update dockerfile spack --- AlmaLinux9/Dockerfile-spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index 0509b73..4674ea0 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -5,7 +5,7 @@ ARG VERSION=release FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build -LABEL org.opencontainers.image.source=https://github.com/madbaron/MuonCollider-docker +LABEL org.opencontainers.image.source=https://github.com/madbaron/mucoll-spack LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" LABEL org.opencontainers.image.licenses=Apache-2.0 From be16f1f14d2a4e7a743dd3f10b98c92d5993c96d Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 14:22:48 +0100 Subject: [PATCH 09/80] change workflow name --- .github/workflows/full-rebuild.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index f07f6a4..fed0404 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -1,5 +1,4 @@ -name: Re-build and publish everything - +name: Re-build and publish all on: push: paths: From 1436eaff2f2b7503866476daa30a2fd15ff5f500 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 14:25:55 +0100 Subject: [PATCH 10/80] test latest commit --- AlmaLinux9/Dockerfile-sim | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 37514e5..156641d 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -8,15 +8,16 @@ ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 # Adding repository: MuColl -ARG MUCOLL_COMMIT=11a32828ca9230337562dc43f91a25862d3a21ff +#ARG MUCOLL_COMMIT=11a32828ca9230337562dc43f91a25862d3a21ff RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ - if [ -n "${MUCOLL_COMMIT}" ]; then \ - cd ${REPOPATH}; \ - git checkout ${MUCOLL_COMMIT}; \ - fi + git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} + # && \ + #if [ -n "${MUCOLL_COMMIT}" ]; then \ + # cd ${REPOPATH}; \ + # git checkout ${MUCOLL_COMMIT}; \ + #fi # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From 640eb537350223e90b5376a59a008672be0fea99 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 14:27:13 +0100 Subject: [PATCH 11/80] trigger full rebuild --- .github/workflows/full-rebuild.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index fed0404..88a73a1 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -3,8 +3,6 @@ on: push: paths: - 'AlmaLinux9/Dockerfile-spack' - tags: - - 'v*' pull_request: branches: - 'master' From 6edfea8586a02ca9cd04d17a24dd58baa6bf0920 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 14:28:22 +0100 Subject: [PATCH 12/80] removed image sourcew --- AlmaLinux9/Dockerfile-spack | 1 - 1 file changed, 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index 4674ea0..b751bb5 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -5,7 +5,6 @@ ARG VERSION=release FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build -LABEL org.opencontainers.image.source=https://github.com/madbaron/mucoll-spack LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" LABEL org.opencontainers.image.licenses=Apache-2.0 From 309871669701922a48c39afc415f02d9647cfcfb Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 14:56:54 +0100 Subject: [PATCH 13/80] update key4hep-spack commit --- AlmaLinux9/Dockerfile-minimal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index 3d2f544..f194bf0 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -8,7 +8,7 @@ ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 # Adding Key4hep repo -ARG KEY4HEP_COMMIT=9f4187ee47dfa1828be936e34c23035818bcd3aa +ARG KEY4HEP_COMMIT=61f91b053eac82a2d56cfe3652c64a001a81a781 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/key4hep-spack && \ From 4348ab06b1f75cbb61d6bbe044ba31031e826cae Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 15:20:01 +0100 Subject: [PATCH 14/80] fix minimal workflow --- .github/workflows/minimal-rebuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index dad256b..6a84a4a 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -3,7 +3,7 @@ name: Re-build and publish Minimal on: push: paths: - - 'AlmaLinux9/Dockerfile-base' + - 'AlmaLinux9/Dockerfile-minimal' workflow_dispatch: env: From 71da3e7171b613e9ad634ebc30b506179dca6b4a Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 15:20:47 +0100 Subject: [PATCH 15/80] update minimal wf --- AlmaLinux9/Dockerfile-minimal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index f194bf0..86db4e1 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -7,7 +7,7 @@ ARG VERSION=main ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 -# Adding Key4hep repo +# Picking specific key4hep commit ARG KEY4HEP_COMMIT=61f91b053eac82a2d56cfe3652c64a001a81a781 RUN source /opt/setup_spack.sh && \ From 4182997d5e6b31e9c71cf88e4bb5d92022bee2ac Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 16:22:14 +0100 Subject: [PATCH 16/80] free even more disk space --- .github/workflows/full-rebuild.yml | 24 ++++++++++++++++++++---- .github/workflows/minimal-rebuild.yml | 24 ++++++++++++++++++++---- .github/workflows/mucoll-rebuild.yml | 12 ++++++++++-- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index 88a73a1..f28783d 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -71,9 +71,17 @@ jobs: IMAGE_NAME: mucoll-minimal steps: - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main + uses: endersonmenezes/free-disk-space@v2 with: - tool-cache: true + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false - name: Checkout uses: actions/checkout@v4 with: @@ -116,9 +124,17 @@ jobs: os: [{dir: AlmaLinux9,suffix: alma9}] steps: - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main + uses: endersonmenezes/free-disk-space@v2 with: - tool-cache: true + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false - name: Checkout uses: actions/checkout@v4 with: diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index 6a84a4a..bb1eae9 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -25,9 +25,17 @@ jobs: IMAGE_NAME: mucoll-minimal steps: - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main + uses: endersonmenezes/free-disk-space@v2 with: - tool-cache: true + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false - name: Checkout uses: actions/checkout@v4 with: @@ -70,9 +78,17 @@ jobs: os: [{dir: AlmaLinux9,suffix: alma9}] steps: - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main + uses: endersonmenezes/free-disk-space@v2 with: - tool-cache: true + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false - name: Checkout uses: actions/checkout@v4 with: diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index 72081dc..8af4e8b 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -27,9 +27,17 @@ jobs: IMAGE_NAME: mucoll-sim steps: - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main + uses: endersonmenezes/free-disk-space@v2 with: - tool-cache: true + remove_android: true + remove_dotnet: true + remove_haskell: true + remove_tool_cache: true + remove_swap: true + remove_packages: "azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox postgresql* temurin-* *llvm* mysql* dotnet-sdk-*" + remove_packages_one_command: true + remove_folders: "/usr/share/swift /usr/share/miniconda /usr/share/az* /usr/share/glade* /usr/local/lib/node_modules /usr/local/share/chromium /usr/local/share/powershell" + testing: false - name: Checkout uses: actions/checkout@v4 with: From 6a2965081e1591c4781ecf269aeea3d513dc2a11 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 16:24:07 +0100 Subject: [PATCH 17/80] updating workflows to self-trigger --- .github/workflows/full-rebuild.yml | 1 + .github/workflows/minimal-rebuild.yml | 1 + .github/workflows/mucoll-rebuild.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index f28783d..760341d 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -2,6 +2,7 @@ name: Re-build and publish all on: push: paths: + - .github/workflows/full-rebuild.yml - 'AlmaLinux9/Dockerfile-spack' pull_request: branches: diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index bb1eae9..0c4d1de 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -3,6 +3,7 @@ name: Re-build and publish Minimal on: push: paths: + - '.github/workflows/minimal-rebuild.yml' - 'AlmaLinux9/Dockerfile-minimal' workflow_dispatch: diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index 8af4e8b..e76f3ed 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -3,6 +3,7 @@ name: Re-build and publish MuColl on: push: paths: + - '.github/workflows/mucoll-rebuild.yml' - 'AlmaLinux9/Dockerfile-sim' - 'environments/**' - 'packages/**' From 7369075e5cae4fa6df20a2b1321ed729c8e9c594 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 20:35:35 +0100 Subject: [PATCH 18/80] checkout correct branch --- .github/workflows/full-rebuild.yml | 2 +- AlmaLinux9/Dockerfile-sim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index 760341d..a711aa2 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -1,4 +1,4 @@ -name: Re-build and publish all +name: Re-build and publish All on: push: paths: diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 156641d..840cd38 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -12,7 +12,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} + git clone -b ${VERSION} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} # && \ #if [ -n "${MUCOLL_COMMIT}" ]; then \ # cd ${REPOPATH}; \ From a1c140522be5db79368314a659702d1428a629dd Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 20:50:59 +0100 Subject: [PATCH 19/80] whitespace --- AlmaLinux9/Dockerfile-sim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 840cd38..4ed86c6 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -3,7 +3,7 @@ # Tag: ${VERSION}-alma9 ############################################################################### -ARG VERSION=dev +ARG VERSION=master ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 @@ -12,7 +12,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone -b ${VERSION} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} + git clone -b ${VERSION} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} # && \ #if [ -n "${MUCOLL_COMMIT}" ]; then \ # cd ${REPOPATH}; \ From 79d2f1f495a7d015e844b41e6126ac37186571a5 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:00:04 +0100 Subject: [PATCH 20/80] version --- AlmaLinux9/Dockerfile-sim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 4ed86c6..254c16b 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -12,7 +12,8 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone -b ${VERSION} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} + echo "Cloning MuColl branch ${VERSION}" && \ + git clone -b "${VERSION}" --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} # && \ #if [ -n "${MUCOLL_COMMIT}" ]; then \ # cd ${REPOPATH}; \ From 58f2f531c902e20e75986897254a0abf5e97f6f4 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:02:44 +0100 Subject: [PATCH 21/80] removing specific podio from debug rel --- environments/mucoll-release-debug/spack.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/environments/mucoll-release-debug/spack.yaml b/environments/mucoll-release-debug/spack.yaml index b599c9e..705b8cc 100644 --- a/environments/mucoll-release-debug/spack.yaml +++ b/environments/mucoll-release-debug/spack.yaml @@ -10,8 +10,6 @@ spack: require: '@1.29 +hepmc3+lcio+xercesc' edm4hep: require: '@0.10.5 ~ipo' - podio: - require: '@0.17.3 ~ipo~rntuple~sio' acts: require: '@32.1.0' whizard: From 987065e420dc5d1538648ceaa7d633a3e3aec0ff Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:06:48 +0100 Subject: [PATCH 22/80] synch debug with rel --- environments/mucoll-release-debug/spack.yaml | 75 +++++++++++++------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/environments/mucoll-release-debug/spack.yaml b/environments/mucoll-release-debug/spack.yaml index 705b8cc..219d4f1 100644 --- a/environments/mucoll-release-debug/spack.yaml +++ b/environments/mucoll-release-debug/spack.yaml @@ -1,27 +1,20 @@ spack: packages: - geant4: - require: '@11.2.0 +opengl+vecgeom' clhep: require: '@2.4.7.1 ~cms~ipo' - root: - require: '@6.32.00 +fftw+http+mlp+pythia8+r+root7+sqlite+ssl+tmva+vc' - dd4hep: - require: '@1.29 +hepmc3+lcio+xercesc' - edm4hep: - require: '@0.10.5 ~ipo' acts: require: '@32.1.0' - whizard: - require: '@3.1.4 +lcio+openloops+openmp+pythia8' - pythia8: - require: '@8.311 ~hepmc' k4simdelphes: require: '@00-06-02 ~delphes_hepmc' - gaudi: - require: '@38.2 +aida+gaudialg' mucoll-stack: - require: '@2.9' + require: '@2.9 +devtools+pytools+ml' + + # The version seems to be necessary, otherwise it defaults to an older version + root: + require: '@6.32: +davix+fftw+gsl+math+minuit+mlp+opengl~postgres~pythia8+python+r+roofit+root7+rpath~shadow+spectrum+sqlite+ssl+tbb+threads+tmva+tmva-cpu+unuran+vc+vdt+x+xml+xrootd cxxstd=20' + + geant4: + require: +opengl+qt~vecgeom cxxstd=20 overlay: require: '@0.25' @@ -40,20 +33,54 @@ spack: muoncvxddigitiser: require: '@0.2' - gear: - require: '@1.9.2' ilcutil: require: '@1.7.2' - marlin: - require: '@1.19.1' - pandorapfa: - require: '@4.8.1' - gdb: - require: '@14.1' + + # Use the system OpenGL as recommended by spack + # See https://github.com/key4hep/key4hep-spack/issues/318 and + # https://github.com/key4hep/key4hep-spack/issues/516 + # A version is needed and we set it to 99 so that spack thinks it's the latest + # (it may not work if the versions in the current system are not what's required + # but that hasn't happened yet) + opengl: + buildable: False + externals: + - spec: opengl@99 + prefix: /usr + openglu: + buildable: False + externals: + - spec: openglu@99 + prefix: /usr + mesa: + buildable: False + externals: + - spec: mesa@99 + prefix: /usr + mesa-glu: + buildable: False + externals: + - spec: mesa-glu@99 + prefix: /usr + # Use the system kerberos, otherwise the tickets obtained at login with the + # system version don't appear for the spack kerberos (happening on lxplus at least) + krb5: + buildable: False + externals: + - spec: krb5@99 + prefix: /usr + # Use the system ssh, otherwise the configuration for the provided ssh will likely be + # different and things like ssh using kerberos won't work (by extension authentication + # for git with ssh with kerberos) + openssh: + buildable: False + externals: + - spec: openssh@99 + prefix: /usr all: target: [x86_64] - variants: build_type=Debug cxxstd=17 + variants: build_type=Debug cxxstd=20 buildable: true providers: gl: [mesa] From 4a0c69ab7b1610c3d3cee9ff9e21cea5a29fe967 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:07:31 +0100 Subject: [PATCH 23/80] capital C --- .github/workflows/concretize.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/concretize.yaml b/.github/workflows/concretize.yaml index d52b424..fadfc1c 100644 --- a/.github/workflows/concretize.yaml +++ b/.github/workflows/concretize.yaml @@ -1,4 +1,4 @@ -name: concretize-ubuntu +name: Concretize-ubuntu on: [push, pull_request] From 7a1a62c48daa990aa89cbef7702b1c69413308b0 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:09:57 +0100 Subject: [PATCH 24/80] checking out master --- AlmaLinux9/Dockerfile-sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 254c16b..db17bff 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -13,7 +13,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ echo "Cloning MuColl branch ${VERSION}" && \ - git clone -b "${VERSION}" --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} + git clone -b master --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} # && \ #if [ -n "${MUCOLL_COMMIT}" ]; then \ # cd ${REPOPATH}; \ From 1e3bd6b7fb68a91900bd04639ed45751b8d95ee9 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:14:54 +0100 Subject: [PATCH 25/80] checking out correct branch --- AlmaLinux9/Dockerfile-sim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index db17bff..808c27e 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -5,6 +5,9 @@ ARG VERSION=master ARG REPOSITORY=madbaron + +ENV BRANCH=${VERSION} + FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 # Adding repository: MuColl @@ -12,8 +15,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - echo "Cloning MuColl branch ${VERSION}" && \ - git clone -b master --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} + git clone -b ${BRANCH} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} # && \ #if [ -n "${MUCOLL_COMMIT}" ]; then \ # cd ${REPOPATH}; \ From 8bc827de7732a8e370291c15d33d9a28fe58c796 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:18:51 +0100 Subject: [PATCH 26/80] updating github actions --- .github/workflows/concretize-template.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/concretize-template.yaml b/.github/workflows/concretize-template.yaml index 6ca3866..fec5ee0 100644 --- a/.github/workflows/concretize-template.yaml +++ b/.github/workflows/concretize-template.yaml @@ -13,19 +13,19 @@ jobs: fail-fast: false steps: - name: Checkout Spack - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: spack/spack path: spack - name: Checkout Key4hep - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: key4hep/key4hep-spack path: key4hep-spack - name: Checkout this repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: mucoll-spack @@ -42,7 +42,6 @@ jobs: spack spec -Nt > ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log - name: Store artifacts - uses: actions/upload-artifact@v3 - with: + uses: actions/upload-artifact@v4 name: spec-list path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log From 8ddca0045b0316ea952ee6866935c57ae36c81cd Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:19:54 +0100 Subject: [PATCH 27/80] fix spec list --- .github/workflows/concretize-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/concretize-template.yaml b/.github/workflows/concretize-template.yaml index fec5ee0..928876f 100644 --- a/.github/workflows/concretize-template.yaml +++ b/.github/workflows/concretize-template.yaml @@ -43,5 +43,6 @@ jobs: - name: Store artifacts uses: actions/upload-artifact@v4 + with: name: spec-list path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log From 602855e1e249805925dcf7196c670c947d57c2d9 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:29:16 +0100 Subject: [PATCH 28/80] passing git hash --- .github/workflows/full-rebuild.yml | 2 ++ .github/workflows/minimal-rebuild.yml | 2 ++ .github/workflows/mucoll-rebuild.yml | 2 ++ AlmaLinux9/Dockerfile-sim | 17 ++++++----------- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index a711aa2..9099960 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -169,3 +169,5 @@ jobs: VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + env: + GIT_HASH: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index 0c4d1de..d3b13ce 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -123,3 +123,5 @@ jobs: VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + env: + GIT_HASH: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index e76f3ed..43ce9a6 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -72,3 +72,5 @@ jobs: VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + env: + GIT_HASH: ${{ github.ref }} diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 808c27e..a26af1d 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -5,22 +5,17 @@ ARG VERSION=master ARG REPOSITORY=madbaron - -ENV BRANCH=${VERSION} - FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 # Adding repository: MuColl -#ARG MUCOLL_COMMIT=11a32828ca9230337562dc43f91a25862d3a21ff - RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone -b ${BRANCH} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} - # && \ - #if [ -n "${MUCOLL_COMMIT}" ]; then \ - # cd ${REPOPATH}; \ - # git checkout ${MUCOLL_COMMIT}; \ - #fi + git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ + if [ -n "${GIT_HASH}" ]; then \ + cd ${REPOPATH}; \ + git checkout ${GIT_HASH}; \ + cd -; \ + fi # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From e4acb5d3fa2ba33cb7dc71c2b6c49009775e3736 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:30:34 +0100 Subject: [PATCH 29/80] passing git hash --- .github/workflows/full-rebuild.yml | 4 ++-- .github/workflows/minimal-rebuild.yml | 4 ++-- .github/workflows/mucoll-rebuild.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index 9099960..94117e8 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -169,5 +169,5 @@ jobs: VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} - env: - GIT_HASH: ${{ github.ref }} \ No newline at end of file + env: + GIT_HASH: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index d3b13ce..de93005 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -123,5 +123,5 @@ jobs: VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} - env: - GIT_HASH: ${{ github.ref }} \ No newline at end of file + env: + GIT_HASH: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index 43ce9a6..d0655f2 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -72,5 +72,5 @@ jobs: VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} - env: - GIT_HASH: ${{ github.ref }} + env: + GIT_HASH: ${{ github.ref }} From 2c8a205635072050910c0c64bf54a1084df30a76 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:39:06 +0100 Subject: [PATCH 30/80] checking out hash --- AlmaLinux9/Dockerfile-sim | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index a26af1d..d3c6376 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -11,11 +11,9 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ - if [ -n "${GIT_HASH}" ]; then \ - cd ${REPOPATH}; \ - git checkout ${GIT_HASH}; \ - cd -; \ - fi + cd ${REPOPATH}; \ + git checkout ${GIT_HASH}; \ + cd -; # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From a71824d92c526791a6ddf193a3e37c4c48484bcb Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 21:54:25 +0100 Subject: [PATCH 31/80] back to branch --- .github/workflows/full-rebuild.yml | 2 +- .github/workflows/minimal-rebuild.yml | 2 +- .github/workflows/mucoll-rebuild.yml | 2 +- AlmaLinux9/Dockerfile-sim | 5 +---- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index 94117e8..f578397 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -170,4 +170,4 @@ jobs: REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} env: - GIT_HASH: ${{ github.ref }} \ No newline at end of file + GIT_BRANCH: ${{ steps.meta.outputs.version }} \ No newline at end of file diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index de93005..cd7dafb 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -124,4 +124,4 @@ jobs: REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} env: - GIT_HASH: ${{ github.ref }} \ No newline at end of file + GIT_BRANCH: ${{ steps.meta.outputs.version }} \ No newline at end of file diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index d0655f2..c07e6d7 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -73,4 +73,4 @@ jobs: REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} env: - GIT_HASH: ${{ github.ref }} + GIT_BRANCH: ${{ steps.meta.outputs.version }} \ No newline at end of file diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index d3c6376..1499e1f 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -10,10 +10,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ - cd ${REPOPATH}; \ - git checkout ${GIT_HASH}; \ - cd -; + git clone -b ${GIT_BRANCH} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From 287a81851d67daff0fd6147c557301d29a85bbc1 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Thu, 21 Nov 2024 22:02:45 +0100 Subject: [PATCH 32/80] back to master --- AlmaLinux9/Dockerfile-sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 1499e1f..ed6f261 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -10,7 +10,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone -b ${GIT_BRANCH} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ + git clone -b master --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From 50ce10f209238cf73bf13bc75b2582c85994d00c Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 22 Nov 2024 11:22:17 +0100 Subject: [PATCH 33/80] fix line return --- AlmaLinux9/Dockerfile-sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index ed6f261..07804de 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -10,7 +10,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone -b master --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ + git clone -b master --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From 26a13de7456a804d5b238131b3a58594beffd60c Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 22 Nov 2024 13:16:07 +0100 Subject: [PATCH 34/80] py-onnx fix --- .cherry-pick | 3 +++ AlmaLinux9/Dockerfile-minimal | 4 ++++ AlmaLinux9/Dockerfile-sim | 4 ++++ AlmaLinux9/Dockerfile-spack | 6 ++++-- environments/mucoll-common/packages.yaml | 4 ++-- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.cherry-pick b/.cherry-pick index e69de29..fb81a4a 100644 --- a/.cherry-pick +++ b/.cherry-pick @@ -0,0 +1,3 @@ +# py-onnx: add a patch for the standard, remove after 1.15.0 +git add . +git cherry-pick bbd205543b09139f2f57f21099eb366c9f183d76 -X theirs --no-commit \ No newline at end of file diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index 86db4e1..9a7557d 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -3,6 +3,10 @@ # Tag: ${VERSION}-alma9 ############################################################################### +LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" +LABEL org.opencontainers.image.licenses=Apache-2.0 +LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" + ARG VERSION=main ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 07804de..d3234c1 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -3,6 +3,10 @@ # Tag: ${VERSION}-alma9 ############################################################################### +LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" +LABEL org.opencontainers.image.licenses=Apache-2.0 +LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" + ARG VERSION=master ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index b751bb5..538968a 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -3,10 +3,12 @@ # Tag: ${VERSION}-alma9 ############################################################################### -ARG VERSION=release -FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" LABEL org.opencontainers.image.licenses=Apache-2.0 +LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" + +ARG VERSION=release +FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build # Add the new Certificate Authority (needed for xrootd) RUN wget https://uit.stanford.edu/sites/default/files/2023/10/11/incommon-rsa-ca2.pem -P /usr/share/pki/ca-trust-source/anchors && \ diff --git a/environments/mucoll-common/packages.yaml b/environments/mucoll-common/packages.yaml index 4987edd..0f1cae9 100644 --- a/environments/mucoll-common/packages.yaml +++ b/environments/mucoll-common/packages.yaml @@ -1,7 +1,7 @@ # Very loose requirements for nightly builds packages: geant4: - variants: ~qt+opengl+vecgeom cxxstd=17 + variants: ~qt+opengl+vecgeom cxxstd=20 root: variants: +davix+fftw+gsl+math+minuit+mlp+opengl~postgres~pythia6+pythia8+python+r+root7+roofit+rpath~shadow+sqlite+ssl+tbb+threads+tmva+unuran+vc+vdt+x+xml+xrootd cxxstd=17 build_type=RelWithDebInfo dd4hep: @@ -14,7 +14,7 @@ packages: variants: ~delphes_hepmc all: target: [x86_64] - variants: build_type=Release cxxstd=17 + variants: build_type=Release cxxstd=20 buildable: true providers: gl: [mesa] From f667a0f47069a53bd4efea61b00b411e12c4e386 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 22 Nov 2024 13:17:27 +0100 Subject: [PATCH 35/80] py-onnx fix for real --- environments/mucoll-release/spack.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 2426290..4092914 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -9,6 +9,9 @@ spack: mucoll-stack: require: '@2.9 +devtools+pytools+ml' + py-setuptools: + require: "@69" + # The version seems to be necessary, otherwise it defaults to an older version root: require: '@6.32: +davix+fftw+gsl+math+minuit+mlp+opengl~postgres~pythia8+python+r+roofit+root7+rpath~shadow+spectrum+sqlite+ssl+tbb+threads+tmva+tmva-cpu+unuran+vc+vdt+x+xml+xrootd cxxstd=20' From 178429897d6883ce2d0ddf6bbfce044d1feaa95b Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 22 Nov 2024 13:18:59 +0100 Subject: [PATCH 36/80] fixing labels --- AlmaLinux9/Dockerfile-minimal | 8 ++++---- AlmaLinux9/Dockerfile-sim | 8 ++++---- AlmaLinux9/Dockerfile-spack | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index 9a7557d..10da429 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -3,14 +3,14 @@ # Tag: ${VERSION}-alma9 ############################################################################### -LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" -LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" - ARG VERSION=main ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 +LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" +LABEL org.opencontainers.image.licenses=Apache-2.0 +LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" + # Picking specific key4hep commit ARG KEY4HEP_COMMIT=61f91b053eac82a2d56cfe3652c64a001a81a781 diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index d3234c1..72eabcb 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -3,14 +3,14 @@ # Tag: ${VERSION}-alma9 ############################################################################### -LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" -LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" - ARG VERSION=master ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 +LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" +LABEL org.opencontainers.image.licenses=Apache-2.0 +LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" + # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index 538968a..a7b8ef8 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -3,13 +3,13 @@ # Tag: ${VERSION}-alma9 ############################################################################### +ARG VERSION=release +FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build + LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" LABEL org.opencontainers.image.licenses=Apache-2.0 LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" -ARG VERSION=release -FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build - # Add the new Certificate Authority (needed for xrootd) RUN wget https://uit.stanford.edu/sites/default/files/2023/10/11/incommon-rsa-ca2.pem -P /usr/share/pki/ca-trust-source/anchors && \ /usr/bin/update-ca-trust From fc2db5aaee8faad3166bbc131353ad210a38e271 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 22 Nov 2024 13:29:25 +0100 Subject: [PATCH 37/80] trying to checkout correct branch --- .github/workflows/full-rebuild.yml | 5 ++--- .github/workflows/minimal-rebuild.yml | 5 ++--- .github/workflows/mucoll-rebuild.yml | 3 +-- AlmaLinux9/Dockerfile-sim | 4 +++- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index f578397..b762341 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -168,6 +168,5 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} - tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} - env: - GIT_BRANCH: ${{ steps.meta.outputs.version }} \ No newline at end of file + MUCOLL_BRANCH=${{ steps.meta.outputs.version }} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} \ No newline at end of file diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index cd7dafb..91bc0fe 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -122,6 +122,5 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} - tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} - env: - GIT_BRANCH: ${{ steps.meta.outputs.version }} \ No newline at end of file + MUCOLL_BRANCH=${{ steps.meta.outputs.version }} + tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} \ No newline at end of file diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index c07e6d7..b2c747f 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -71,6 +71,5 @@ jobs: build-args: | VERSION=${{ steps.meta.outputs.version }} REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} + MUCOLL_BRANCH=${{ steps.meta.outputs.version }} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} - env: - GIT_BRANCH: ${{ steps.meta.outputs.version }} \ No newline at end of file diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 72eabcb..51ab5a6 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -11,10 +11,12 @@ LABEL org.opencontainers.image.description="Muon collider software stack based o LABEL org.opencontainers.image.licenses=Apache-2.0 LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" +ARG MUCOLL_BRANCH=master + # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone -b master --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} + git clone -b ${MUCOLL_BRANCH} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From aac46d0896bad70d045e016c76063ff5f6fac0a5 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 22 Nov 2024 14:53:58 +0100 Subject: [PATCH 38/80] fix to pick up the right version of fmt --- AlmaLinux9/Dockerfile-minimal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index 10da429..ffde8f9 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -12,7 +12,7 @@ LABEL org.opencontainers.image.licenses=Apache-2.0 LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" # Picking specific key4hep commit -ARG KEY4HEP_COMMIT=61f91b053eac82a2d56cfe3652c64a001a81a781 +ARG KEY4HEP_COMMIT=475d26f0a62283a29e87b38399b128798d2b81a8 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/key4hep-spack && \ From 96240e5e5b250f93e82bbc976825edf78b2291ab Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sat, 23 Nov 2024 20:36:56 +0100 Subject: [PATCH 39/80] build with gcc14 --- AlmaLinux9/Dockerfile-spack | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index a7b8ef8..aa39124 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -28,6 +28,10 @@ RUN if [ -n "${SPACK_COMMIT}" ]; then \ git checkout ${SPACK_COMMIT}; \ fi +RUN source /opt/setup_spack.sh && \ + spack install gcc@14.2.0 && \ + spack clean -a + # Registering compilers with Spack ENV SPACK_COLOR="always" RUN source /opt/setup_spack.sh && \ From acaba8310cddaa1c5ecc9e201f03f0a5176141db Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sat, 23 Nov 2024 20:38:50 +0100 Subject: [PATCH 40/80] removed separate concretisation test --- .github/workflows/concretize-template.yaml | 48 ---------------------- .github/workflows/concretize.yaml | 14 ------- 2 files changed, 62 deletions(-) delete mode 100644 .github/workflows/concretize-template.yaml delete mode 100644 .github/workflows/concretize.yaml diff --git a/.github/workflows/concretize-template.yaml b/.github/workflows/concretize-template.yaml deleted file mode 100644 index 928876f..0000000 --- a/.github/workflows/concretize-template.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: concretize-template - -on: - workflow_call: - inputs: - target: - required: true - type: string -jobs: - concretize: - runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - - name: Checkout Spack - uses: actions/checkout@v4 - with: - repository: spack/spack - path: spack - - - name: Checkout Key4hep - uses: actions/checkout@v4 - with: - repository: key4hep/key4hep-spack - path: key4hep-spack - - - name: Checkout this repo - uses: actions/checkout@v4 - with: - path: mucoll-spack - - - name: Concretize mucoll-common - run: | - source spack/share/spack/setup-env.sh - spack --version - spack repo add key4hep-spack - spack repo add mucoll-spack - spack compiler find - spack env activate mucoll-spack/environments/mucoll-${{ inputs.target }} - spack add mucoll-stack %gcc@11: - spack concretize - spack spec -Nt > ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log - - - name: Store artifacts - uses: actions/upload-artifact@v4 - with: - name: spec-list - path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log diff --git a/.github/workflows/concretize.yaml b/.github/workflows/concretize.yaml deleted file mode 100644 index fadfc1c..0000000 --- a/.github/workflows/concretize.yaml +++ /dev/null @@ -1,14 +0,0 @@ -name: Concretize-ubuntu - -on: [push, pull_request] - -jobs: - concretize: - strategy: - fail-fast: false - matrix: - target: [common, release, release-debug] - uses: ./.github/workflows/concretize-template.yaml - with: - target: ${{ matrix.target }} - From ba1d0016aea4be4d86090378362a3eb9aea415ad Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sat, 23 Nov 2024 21:04:00 +0100 Subject: [PATCH 41/80] starting from more recent image --- AlmaLinux9/Dockerfile-spack | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index aa39124..fd96075 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -4,12 +4,22 @@ ############################################################################### ARG VERSION=release -FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build +FROM gitlab-registry.cern.ch/linuxsupport/alma9-base LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" LABEL org.opencontainers.image.licenses=Apache-2.0 LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" +# epel-release and parallel needed for the validation +# wget needed for some tests +RUN dnf update -y && \ + dnf groupinstall -y "Development Tools" && \ + dnf install -y vim gfortran wget python3-pip epel-release mesa-libGL mesa-libGL-devel mesa-libGLU mesa-libGLU-devel krb5-devel && \ + dnf install -y parallel ccache mold && \ + dnf clean all + +RUN pip install boto3 requests + # Add the new Certificate Authority (needed for xrootd) RUN wget https://uit.stanford.edu/sites/default/files/2023/10/11/incommon-rsa-ca2.pem -P /usr/share/pki/ca-trust-source/anchors && \ /usr/bin/update-ca-trust From 34b778cf65d0b27b119f88f6b314958e1357d9ab Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 13:11:38 +0100 Subject: [PATCH 42/80] specifying gaudi options --- environments/mucoll-release/spack.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 4092914..37b27f1 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -38,7 +38,9 @@ spack: ilcutil: require: '@1.7.2' - + gaudi: + require: +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc + # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and # https://github.com/key4hep/key4hep-spack/issues/516 From e8773fd706d99f5572034221dbd2c36757474934 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 15:31:27 +0100 Subject: [PATCH 43/80] trying pick up gcc14 --- environments/mucoll-common/compilers.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 environments/mucoll-common/compilers.yaml diff --git a/environments/mucoll-common/compilers.yaml b/environments/mucoll-common/compilers.yaml new file mode 100644 index 0000000..f2d0bfa --- /dev/null +++ b/environments/mucoll-common/compilers.yaml @@ -0,0 +1,14 @@ +compilers: +- compiler: + spec: gcc@=14.2.0 + paths: + cc: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/gcc + cxx: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/g++ + f77: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/gfortran + fc: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/gfortran + flags: {} + operating_system: almalinux9 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] \ No newline at end of file From 3d7b4951f4fe1ea9c3f87aabf11a4f36c45b3ddb Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 15:31:55 +0100 Subject: [PATCH 44/80] trying pick up gcc14 --- AlmaLinux9/Dockerfile-minimal | 1 - 1 file changed, 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index ffde8f9..1eac67b 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -2,7 +2,6 @@ # Repository: ${REPOSITORY}/key4hep-base # Tag: ${VERSION}-alma9 ############################################################################### - ARG VERSION=main ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 From 57bacb30903dab3201d376cd4e4683e43f5a79a2 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 15:41:39 +0100 Subject: [PATCH 45/80] adding compilers? --- environments/mucoll-common/spack.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/environments/mucoll-common/spack.yaml b/environments/mucoll-common/spack.yaml index 372d332..1548d37 100644 --- a/environments/mucoll-common/spack.yaml +++ b/environments/mucoll-common/spack.yaml @@ -1,3 +1,4 @@ spack: include: - packages.yaml + - compilers.yaml From f03a421a4c1d739955e141d093593d4bfd6715b1 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 15:42:27 +0100 Subject: [PATCH 46/80] adding compilers --- environments/mucoll-release/spack.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 37b27f1..3f8b8e1 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -1,4 +1,7 @@ spack: + include: + - ../mucoll-common/compilers.yaml + packages: clhep: require: '@2.4.7.1 ~cms~ipo' From f1afb7b8f673253f5381905a4cdd0fc14553b153 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 18:56:40 +0100 Subject: [PATCH 47/80] picking up fixed gaudi --- AlmaLinux9/Dockerfile-minimal | 1 + environments/mucoll-release/spack.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index 1eac67b..ffde8f9 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -2,6 +2,7 @@ # Repository: ${REPOSITORY}/key4hep-base # Tag: ${VERSION}-alma9 ############################################################################### + ARG VERSION=main ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 3f8b8e1..763c6b8 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -42,7 +42,7 @@ spack: ilcutil: require: '@1.7.2' gaudi: - require: +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc + require: '@39.1 +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and From 8eb7d6db9d91661b76d9d173619e42056b54d3e4 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 21:33:42 +0100 Subject: [PATCH 48/80] picking up fixed gaudi --- environments/mucoll-release/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 763c6b8..00aba01 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -42,7 +42,7 @@ spack: ilcutil: require: '@1.7.2' gaudi: - require: '@39.1 +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' + require: '@39.0 +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and From ad3f1c75fbe0dcbc0aad5127d10556eec07b776f Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 21:53:38 +0100 Subject: [PATCH 49/80] adding gaudi 39 --- .cherry-pick | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.cherry-pick b/.cherry-pick index fb81a4a..6af7e4b 100644 --- a/.cherry-pick +++ b/.cherry-pick @@ -1,3 +1,6 @@ # py-onnx: add a patch for the standard, remove after 1.15.0 git add . -git cherry-pick bbd205543b09139f2f57f21099eb366c9f183d76 -X theirs --no-commit \ No newline at end of file +git cherry-pick bbd205543b09139f2f57f21099eb366c9f183d76 -X theirs --no-commit + +# add gaudi 39 +curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/46572.diff | patch -p1 \ No newline at end of file From 648754f12f0d78363683b6cd0d24f42e8fae7519 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 21:55:25 +0100 Subject: [PATCH 50/80] null --- AlmaLinux9/Dockerfile-sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 51ab5a6..945e33f 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -7,7 +7,7 @@ ARG VERSION=master ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 -LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" +LABEL org.opencontainers.image.description="Muon collider software stack" LABEL org.opencontainers.image.licenses=Apache-2.0 LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" From 9ba728874ffd3f3f9bd61195bfb69f26ccf49fb7 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Sun, 24 Nov 2024 22:12:34 +0100 Subject: [PATCH 51/80] mini cleanup --- .cherry-pick | 3 +-- AlmaLinux9/Dockerfile-sim | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.cherry-pick b/.cherry-pick index 6af7e4b..5f1f2b0 100644 --- a/.cherry-pick +++ b/.cherry-pick @@ -1,6 +1,5 @@ # py-onnx: add a patch for the standard, remove after 1.15.0 -git add . -git cherry-pick bbd205543b09139f2f57f21099eb366c9f183d76 -X theirs --no-commit +curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/46571.diff | patch -p1 # add gaudi 39 curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/46572.diff | patch -p1 \ No newline at end of file diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 945e33f..54f61b5 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -41,7 +41,7 @@ RUN source /opt/setup_spack.sh && \ # Concretizing the MuColl stack reusing system packages as external RUN source ${HOME}/setup_env.sh && \ - spack add mucoll-stack && \ + # spack add mucoll-stack && \ spack concretize --reuse # Installing fragments of dependency tree in separate layers for cached debugging From 34bbe526f4ba10be44d30a2ee25b302c8b936bfc Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 09:17:59 +0100 Subject: [PATCH 52/80] revert patch --- .cherry-pick | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cherry-pick b/.cherry-pick index 5f1f2b0..f697f1e 100644 --- a/.cherry-pick +++ b/.cherry-pick @@ -1,5 +1,5 @@ # py-onnx: add a patch for the standard, remove after 1.15.0 -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/46571.diff | patch -p1 +git cherry-pick bbd205543b09139f2f57f21099eb366c9f183d76 -X theirs --no-commit # add gaudi 39 -curl -s https://patch-diff.githubusercontent.com/raw/spack/spack/pull/46572.diff | patch -p1 \ No newline at end of file +git cherry-pick 5ebf45861f688337dddcdee2242d4f479a832d06 -X theirs --no-commit \ No newline at end of file From 6a008c8e2b1eae707eb85006c05a6cc6b4bf31dd Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 09:18:58 +0100 Subject: [PATCH 53/80] make workflow aware of cherry pick --- .github/workflows/mucoll-rebuild.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index b2c747f..bc53e86 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -4,6 +4,7 @@ on: push: paths: - '.github/workflows/mucoll-rebuild.yml' + - '.cherry-pick' - 'AlmaLinux9/Dockerfile-sim' - 'environments/**' - 'packages/**' From e744855bdb40779691d939dce64daf84f507e97d Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 09:26:00 +0100 Subject: [PATCH 54/80] debug printout --- AlmaLinux9/Dockerfile-sim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 54f61b5..4cfe67f 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -22,7 +22,8 @@ RUN source /opt/setup_spack.sh && \ COPY apply_patches.sh ./apply_patches.sh RUN source /opt/setup_spack.sh && \ - ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack + ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack && \ + cat var/spack/repos/builtin/packages/gaudi/package.py # Add the package repositories RUN source /opt/setup_spack.sh && \ From c2276312fb7287d2c5314ff07d1657f870427fa1 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 09:33:19 +0100 Subject: [PATCH 55/80] fix debug printout --- AlmaLinux9/Dockerfile-sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 4cfe67f..488670e 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -23,7 +23,7 @@ COPY apply_patches.sh ./apply_patches.sh RUN source /opt/setup_spack.sh && \ ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack && \ - cat var/spack/repos/builtin/packages/gaudi/package.py + cat /opt/spack/var/spack/repos/builtin/packages/gaudi/package.py # Add the package repositories RUN source /opt/setup_spack.sh && \ From c1dbecce4cc5beed8c5c1a66b00d8d4c7b43faf5 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 09:43:41 +0100 Subject: [PATCH 56/80] removing explicit gaudi version --- AlmaLinux9/Dockerfile-sim | 2 +- environments/mucoll-release/spack.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 488670e..6d2bbbf 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -9,7 +9,7 @@ FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 LABEL org.opencontainers.image.description="Muon collider software stack" LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" +LABEL org.opencontainers.image.source="https://github.com/madbaron/mucoll-spack" ARG MUCOLL_BRANCH=master diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 00aba01..333e450 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -42,7 +42,7 @@ spack: ilcutil: require: '@1.7.2' gaudi: - require: '@39.0 +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' + require: '+aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and From 071a7e9c75de46cb0653a160dfe91db95174f5d7 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 09:52:12 +0100 Subject: [PATCH 57/80] trying just gaudi@39 --- environments/mucoll-release/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 333e450..a73baff 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -42,7 +42,7 @@ spack: ilcutil: require: '@1.7.2' gaudi: - require: '+aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' + require: '@39 +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and From 04f656ad43dfe1f8ce9cecb2c497c570c0ebaaa5 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 09:53:58 +0100 Subject: [PATCH 58/80] got rid of rogue gaudi folder --- packages/gaudi/package.py | 138 -------------------------------------- 1 file changed, 138 deletions(-) delete mode 100644 packages/gaudi/package.py diff --git a/packages/gaudi/package.py b/packages/gaudi/package.py deleted file mode 100644 index f8d906a..0000000 --- a/packages/gaudi/package.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.package import * -from spack.pkg.builtin.boost import Boost - - -class Gaudi(CMakePackage): - """An experiment-independent HEP event data processing framework""" - - homepage = "https://gaudi.web.cern.ch/gaudi/" - git = "https://gitlab.cern.ch/gaudi/Gaudi.git" - url = "https://gitlab.cern.ch/gaudi/Gaudi/-/archive/v33r1/Gaudi-v33r1.tar.gz" - - tags = ["hep"] - - version("master", branch="master") - version("38.2", sha256="08759b1398336987ad991602e37079f0744e8d8e4e3d5df2d253b8dedf925068") - version("38.1", sha256="79d42833edcebc2099f91badb6f72708640c05f678cc4521a86e857f112486dc") - version("38.0", sha256="52f2733fa0af760c079b3438bb9c7e36b28ea704f78b0085458e1918c11e1653") - version("37.2", sha256="9b866caab46e182de98b59eddbde80d6fa0e670fe4a35906f1518b04bd99b2d2") - version("37.1", sha256="1d7038fd5dfb5f2517ce57623cf8090549ffe2ea8f0171d534e5c1ca20bd009a") - version("37.0", sha256="823f3821a4f498ddd2dd123fbb8a3787b361ddfd818f4ab13572076fc9afdfe4") - version("36.14", sha256="b11e0afcb797d61a305856dfe8079d48d74c6b6867ceccc0a83aab5978c9ba5f") - version("36.13", sha256="41e711c83428663996c825044b268ce515bef85dad74b4a9453f2207b4b1be7b") - version("36.12", sha256="dfce9156cedfa0a7234f880a3c395e592a5f3dc79070d5d196fdb94b83ae203e") - version("36.11", sha256="81664d033b0aa8598a0e4cb7e455e697baeb063a11bbde2390164776238ba9f7") - version("36.10", sha256="2c1f181c54a76b493b913aeecbd6595236afc08e41d7f1d80be6fe65ac95adb3") - version("36.9", sha256="b4e080094771f111bd0bcdf744bcab7b028c7e2af7c5dfaa4a977ebbf0160a8f") - version("36.8", sha256="64b4300a57335af7c1f74c736d7610041a1ef0c1f976e3342a22385b60519afc") - version("36.7", sha256="8dca43185ba11e1b33f5535d2e384542d84500407b0d1f8cb920be00f05c9716") - version("36.6", sha256="8fc7be0ce32f99cc6b0be4ebbb246f4bb5008ffbf0c012cb39c0aff813dce6af") - version("36.5", sha256="593e0316118411a5c5fde5d4d87cbfc3d2bb748a8c72a66f4025498fcbdb0f7e") - version("36.4", sha256="1a5c27cdc21ec136b47f5805406c92268163393c821107a24dbb47bd88e4b97d") - version("36.3", sha256="9ac228d8609416afe4dea6445c6b3ccebac6fab1e46121fcc3a056e24a5d6640") - version("36.2", sha256="a1b4bb597941a7a5b8d60382674f0b4ca5349c540471cd3d4454efbe7b9a09b9") - version("36.1", sha256="9f718c832313676249e5c3ac76ba4346978ee2328f8cdcb29176498b080402e9") - version("36.0", sha256="8a0458cef5b616532f9db7cca9fa0e892e602b64c9e93dc0cc6d972e03034830") - version("35.0", sha256="c01b822f9592a7bf875b9997cbeb3c94dea97cb13d523c12649dbbf5d69b5fa6") - - maintainers("drbenmorgan", "vvolkl", "jmcarcell") - - variant("aida", default=False, description="Build AIDA interfaces support") - variant("cppunit", default=False, description="Build with CppUnit unit testing") - variant("docs", default=False, description="Build documentation with Doxygen") - variant("examples", default=False, description="Build examples") - variant("gaudialg", default=False, description="Build GaudiAlg support", when="@37.0:") - variant("gperftools", default=False, description="Build with Google PerfTools support") - variant("heppdt", default=False, description="Build with HEP Particle Data Table support") - variant("jemalloc", default=False, description="Build with jemalloc allocator support") - variant("unwind", default=False, description="Build with unwind call-chains") - variant("vtune", default=False, description="Build with Intel VTune profiler support") - variant("xercesc", default=False, description="Build with Xerces-C XML support") - - # These dependencies are needed for a minimal Gaudi build - depends_on("aida") - depends_on("boost@1.67.0: +python") - - # TODO: replace this with an explicit list of components of Boost, - # for instance depends_on('boost +filesystem') - # See https://github.com/spack/spack/pull/22303 for reference - depends_on(Boost.with_default_variants) - depends_on("clhep") - depends_on("cmake", type="build") - depends_on("cppgsl") - depends_on("fmt") - depends_on("fmt@:8", when="@:36.9") - depends_on("intel-tbb@:2020.3", when="@:37.0") - depends_on("tbb", when="@37.1:") - depends_on("uuid") - depends_on("nlohmann-json") - depends_on("python", type=("build", "run")) - depends_on("py-networkx", type=("build", "run")) - depends_on("py-six", type=("build", "run")) - depends_on("py-pyyaml", type=("build", "run", "test")) - depends_on("range-v3") - depends_on("root +python +root7 +ssl +tbb +threads") - depends_on("zlib-api") - - # Testing dependencies - # Note: gaudi only builds examples when testing enabled - for pv in (["catch2", "@36.8:"], ["py-nose", "@35:"], ["py-pytest", "@36.2:"]): - depends_on(pv[0], when=pv[1], type="test") - depends_on(pv[0], when=pv[1] + " +examples") - - # Adding these dependencies triggers the build of most optional components - depends_on("cppgsl", when="+cppunit") - depends_on("cppunit", when="+cppunit") - depends_on("doxygen +graphviz", when="+docs") - depends_on("gperftools", when="+gperftools") - depends_on("gdb") - depends_on("heppdt", when="+heppdt") - depends_on("jemalloc", when="+jemalloc") - depends_on("libunwind", when="+unwind") - depends_on("xerces-c", when="+xercesc") - # NOTE: pocl cannot be added as a minimal OpenCL implementation because - # ROOT does not like being exposed to LLVM symbols. - - # The Intel VTune dependency is taken aside because it requires a license - depends_on("intel-parallel-studio -mpi +vtune", when="+vtune") - - def cmake_args(self): - args = [ - # Note: gaudi only builds examples when testing enabled - self.define("BUILD_TESTING", self.run_tests or self.spec.satisfies("+examples")), - self.define_from_variant("GAUDI_USE_AIDA", "aida"), - self.define_from_variant("GAUDI_USE_CPPUNIT", "cppunit"), - self.define_from_variant("GAUDI_ENABLE_GAUDIALG", "gaudialg"), - self.define_from_variant("GAUDI_USE_GPERFTOOLS", "gperftools"), - self.define_from_variant("GAUDI_USE_HEPPDT", "heppdt"), - self.define_from_variant("GAUDI_USE_JEMALLOC", "jemalloc"), - self.define_from_variant("GAUDI_USE_UNWIND", "unwind"), - self.define_from_variant("GAUDI_USE_XERCESC", "xercesc"), - self.define_from_variant("GAUDI_USE_DOXYGEN", "docs"), - # needed to build core services like rndmsvc - self.define("GAUDI_USE_CLHEP", True), - self.define("GAUDI_USE_PYTHON_MAJOR", str(self.spec["python"].version.up_to(1))), - # todo: - self.define("GAUDI_USE_INTELAMPLIFIER", False), - ] - return args - - def setup_run_environment(self, env): - # environment as in Gaudi.xenv - env.prepend_path("PATH", self.prefix.scripts) - env.prepend_path("PYTHONPATH", self.prefix.python) - env.prepend_path("LD_LIBRARY_PATH", self.prefix.lib) - env.prepend_path("LD_LIBRARY_PATH", self.prefix.lib64) - - def url_for_version(self, version): - major = str(version[0]) - minor = str(version[1]) - url = "https://gitlab.cern.ch/gaudi/Gaudi/-/archive/v{0}r{1}/Gaudi-v{0}r{1}.tar.gz".format( - major, minor - ) - return url From e1b21dcbbb6a006becda9ffd7df009b491f2ebe6 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 12:25:55 +0100 Subject: [PATCH 59/80] back to gcc11.4.1 --- AlmaLinux9/Dockerfile-sim | 5 ++--- environments/mucoll-release/spack.yaml | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 6d2bbbf..7aad2bb 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -22,8 +22,7 @@ RUN source /opt/setup_spack.sh && \ COPY apply_patches.sh ./apply_patches.sh RUN source /opt/setup_spack.sh && \ - ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack && \ - cat /opt/spack/var/spack/repos/builtin/packages/gaudi/package.py + ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack # Add the package repositories RUN source /opt/setup_spack.sh && \ @@ -50,7 +49,7 @@ ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" RUN source ${HOME}/setup_env.sh && \ spack spec -NIt && \ - spack install ${SPACK_INSTALL_OPTS} && \ + spack install ${SPACK_INSTALL_OPTS} gaudi && \ spack clean -a RUN source ${HOME}/setup_env.sh && \ diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index a73baff..6d39e9b 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -1,6 +1,6 @@ spack: - include: - - ../mucoll-common/compilers.yaml + # include: + # - ../mucoll-common/compilers.yaml packages: clhep: @@ -42,7 +42,7 @@ spack: ilcutil: require: '@1.7.2' gaudi: - require: '@39 +aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' + require: '+aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and From ba6c5f9c524414643695a96a33fec0187a551fe2 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 12:29:43 +0100 Subject: [PATCH 60/80] removing gaudi spec --- environments/mucoll-release/spack.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 6d39e9b..9e716eb 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -41,8 +41,6 @@ spack: ilcutil: require: '@1.7.2' - gaudi: - require: '+aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and From 73c5de5916ccc9c1abc8ffc5c5a2dd0b4521b7a9 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 12:45:13 +0100 Subject: [PATCH 61/80] pick up CVXD master --- AlmaLinux9/Dockerfile-sim | 2 +- environments/mucoll-release/spack.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 7aad2bb..c8a9424 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -49,7 +49,7 @@ ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" RUN source ${HOME}/setup_env.sh && \ spack spec -NIt && \ - spack install ${SPACK_INSTALL_OPTS} gaudi && \ + spack install ${SPACK_INSTALL_OPTS} && \ spack clean -a RUN source ${HOME}/setup_env.sh && \ diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 9e716eb..ad5785f 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -37,10 +37,12 @@ spack: actstracking: require: '@1.2.2' muoncvxddigitiser: - require: '@0.2' + require: '@master' ilcutil: require: '@1.7.2' + gaudi: + require: '+aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and From 9965d8cee87a573e7ffd6bf766500ae0d0b35597 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 12:46:09 +0100 Subject: [PATCH 62/80] removing gaudi spec --- environments/mucoll-release/spack.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index ad5785f..7f2b558 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -41,9 +41,7 @@ spack: ilcutil: require: '@1.7.2' - gaudi: - require: '+aida+cppunit+examples+gperftools+heppdt+jemalloc+unwind+xercesc' - + # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and # https://github.com/key4hep/key4hep-spack/issues/516 From ae12a5ac8713851b7997d497b69a38c9116e4bff Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 12:48:51 +0100 Subject: [PATCH 63/80] redo without gcc14 --- AlmaLinux9/Dockerfile-minimal | 2 +- AlmaLinux9/Dockerfile-spack | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index ffde8f9..a14820a 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -9,7 +9,7 @@ FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" +LABEL org.opencontainers.image.source="https://github.com/madbaron/mucoll-spack" # Picking specific key4hep commit ARG KEY4HEP_COMMIT=475d26f0a62283a29e87b38399b128798d2b81a8 diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index fd96075..bcd8de3 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -8,7 +8,7 @@ FROM gitlab-registry.cern.ch/linuxsupport/alma9-base LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source = "https://github.com/madbaron/mucoll-spack" +LABEL org.opencontainers.image.source="https://github.com/madbaron/mucoll-spack" # epel-release and parallel needed for the validation # wget needed for some tests @@ -38,9 +38,9 @@ RUN if [ -n "${SPACK_COMMIT}" ]; then \ git checkout ${SPACK_COMMIT}; \ fi -RUN source /opt/setup_spack.sh && \ - spack install gcc@14.2.0 && \ - spack clean -a +#RUN source /opt/setup_spack.sh && \ +# spack install gcc@14.2.0 && \ +# spack clean -a # Registering compilers with Spack ENV SPACK_COLOR="always" From 3a04dbd1e57911e2676919068ebecfbdabe5c623 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 15:51:59 +0100 Subject: [PATCH 64/80] more recent packages --- environments/mucoll-release/spack.yaml | 10 ++-------- packages/mucoll-stack/package.py | 2 ++ 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 7f2b558..685dcb8 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -3,14 +3,11 @@ spack: # - ../mucoll-common/compilers.yaml packages: - clhep: - require: '@2.4.7.1 ~cms~ipo' acts: require: '@32.1.0' - k4simdelphes: - require: '@00-06-02 ~delphes_hepmc' + mucoll-stack: - require: '@2.9 +devtools+pytools+ml' + require: '@master +devtools+pytools+ml' py-setuptools: require: "@69" @@ -39,9 +36,6 @@ spack: muoncvxddigitiser: require: '@master' - ilcutil: - require: '@1.7.2' - # Use the system OpenGL as recommended by spack # See https://github.com/key4hep/key4hep-spack/issues/318 and # https://github.com/key4hep/key4hep-spack/issues/516 diff --git a/packages/mucoll-stack/package.py b/packages/mucoll-stack/package.py index 439715e..5e624d1 100644 --- a/packages/mucoll-stack/package.py +++ b/packages/mucoll-stack/package.py @@ -26,6 +26,8 @@ class MucollStack(BundlePackage, Key4hepPackage): # should use `environments/mucoll-common/packages.yaml` version(datetime.today().strftime('%Y-%m-%d')) + version("master", branch="master") + ### stable build # to install exact specified version for every dependecy # should use `environments/mucoll-release/packages.yaml` From a9a089c1a63ba26847456d0a3cfe9f2b57d93439 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 15:52:49 +0100 Subject: [PATCH 65/80] shortcut for debug --- AlmaLinux9/Dockerfile-sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index c8a9424..ef856e4 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -49,7 +49,7 @@ ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" RUN source ${HOME}/setup_env.sh && \ spack spec -NIt && \ - spack install ${SPACK_INSTALL_OPTS} && \ + spack install ${SPACK_INSTALL_OPTS} k4simdelphes && \ spack clean -a RUN source ${HOME}/setup_env.sh && \ From 05cd38329013de0a821847628b3e8d4a2b3ed1d8 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 17:01:18 +0100 Subject: [PATCH 66/80] removing compilers from mucoll-common --- AlmaLinux9/Dockerfile-sim | 2 +- environments/mucoll-common/compilers.yaml | 14 -------------- environments/mucoll-common/spack.yaml | 1 - 3 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 environments/mucoll-common/compilers.yaml diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index ef856e4..c8a9424 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -49,7 +49,7 @@ ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" RUN source ${HOME}/setup_env.sh && \ spack spec -NIt && \ - spack install ${SPACK_INSTALL_OPTS} k4simdelphes && \ + spack install ${SPACK_INSTALL_OPTS} && \ spack clean -a RUN source ${HOME}/setup_env.sh && \ diff --git a/environments/mucoll-common/compilers.yaml b/environments/mucoll-common/compilers.yaml deleted file mode 100644 index f2d0bfa..0000000 --- a/environments/mucoll-common/compilers.yaml +++ /dev/null @@ -1,14 +0,0 @@ -compilers: -- compiler: - spec: gcc@=14.2.0 - paths: - cc: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/gcc - cxx: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/g++ - f77: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/gfortran - fc: /opt/spack/opt/spack/linux-almalinux9-zen2/gcc-11.4.1/gcc-14.2.0-dk7egbifzphumnzx4frgxe5yyk53oo6f/bin/gfortran - flags: {} - operating_system: almalinux9 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] \ No newline at end of file diff --git a/environments/mucoll-common/spack.yaml b/environments/mucoll-common/spack.yaml index 1548d37..372d332 100644 --- a/environments/mucoll-common/spack.yaml +++ b/environments/mucoll-common/spack.yaml @@ -1,4 +1,3 @@ spack: include: - packages.yaml - - compilers.yaml From 3d15e514bb6d138688d58cddd24223fbc8c8c5f8 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 17:52:22 +0100 Subject: [PATCH 67/80] removing hardcoded labels --- AlmaLinux9/Dockerfile-minimal | 4 ---- AlmaLinux9/Dockerfile-sim | 4 ---- AlmaLinux9/Dockerfile-spack | 4 ---- 3 files changed, 12 deletions(-) diff --git a/AlmaLinux9/Dockerfile-minimal b/AlmaLinux9/Dockerfile-minimal index a14820a..0512736 100644 --- a/AlmaLinux9/Dockerfile-minimal +++ b/AlmaLinux9/Dockerfile-minimal @@ -7,10 +7,6 @@ ARG VERSION=main ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 -LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" -LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source="https://github.com/madbaron/mucoll-spack" - # Picking specific key4hep commit ARG KEY4HEP_COMMIT=475d26f0a62283a29e87b38399b128798d2b81a8 diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index c8a9424..ff5f941 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -7,10 +7,6 @@ ARG VERSION=master ARG REPOSITORY=madbaron FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 -LABEL org.opencontainers.image.description="Muon collider software stack" -LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source="https://github.com/madbaron/mucoll-spack" - ARG MUCOLL_BRANCH=master # Adding repository: MuColl diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index bcd8de3..096d1b5 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -6,10 +6,6 @@ ARG VERSION=release FROM gitlab-registry.cern.ch/linuxsupport/alma9-base -LABEL org.opencontainers.image.description="Muon collider software stack based on Spack" -LABEL org.opencontainers.image.licenses=Apache-2.0 -LABEL org.opencontainers.image.source="https://github.com/madbaron/mucoll-spack" - # epel-release and parallel needed for the validation # wget needed for some tests RUN dnf update -y && \ From 1c4ba29cebeb3e9744626ffc0d04e6ac3f22265e Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 18:02:57 +0100 Subject: [PATCH 68/80] moving py-onnx* to +ml --- packages/mucoll-stack/package.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/mucoll-stack/package.py b/packages/mucoll-stack/package.py index 5e624d1..5785244 100644 --- a/packages/mucoll-stack/package.py +++ b/packages/mucoll-stack/package.py @@ -112,10 +112,15 @@ class MucollStack(BundlePackage, Key4hepPackage): depends_on('ninja') depends_on('doxygen') depends_on('gdb') - + depends_on('llvm', when='+llvm') - depends_on('onnx', when='+ml') - depends_on('xgboost', when='+ml') + + with when('+ml'): + # ML tools + depends_on('onnx') + depends_on('xgboost') + depends_on('py-onnxruntime') + depends_on('py-onnx') with when('+pytools'): # Python tools @@ -123,8 +128,6 @@ class MucollStack(BundlePackage, Key4hepPackage): depends_on('py-ipython') depends_on('py-jupytext') depends_on('py-matplotlib') - depends_on('py-onnxruntime') - depends_on('py-onnx') depends_on('py-pandas') depends_on('py-particle') depends_on('py-pip') From cbcab9747761153c0b667fac21c2d1d4f5910145 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 21:49:59 +0100 Subject: [PATCH 69/80] removed hardcoded madbaron --- .github/workflows/mucoll-rebuild.yml | 5 +++-- AlmaLinux9/Dockerfile-sim | 14 +++++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mucoll-rebuild.yml b/.github/workflows/mucoll-rebuild.yml index bc53e86..bf0a0c2 100644 --- a/.github/workflows/mucoll-rebuild.yml +++ b/.github/workflows/mucoll-rebuild.yml @@ -71,6 +71,7 @@ jobs: file: ./${{matrix.os.dir}}/Dockerfile-sim build-args: | VERSION=${{ steps.meta.outputs.version }} - REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} - MUCOLL_BRANCH=${{ steps.meta.outputs.version }} + IMAGE=${{ env.REGISTRY }}/${{env.OWNER_LC}} + REPOSITORY=${{ github.repository }} + MUCOLL_SHA=${{ github.sha }} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/${{ env.IMAGE_NAME}}:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index ff5f941..5e01c21 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -1,18 +1,22 @@ ############################################################################### -# Repository: ${REPOSITORY}/mucoll-sim +# Repository: ${IMAGE}/mucoll-sim # Tag: ${VERSION}-alma9 ############################################################################### ARG VERSION=master -ARG REPOSITORY=madbaron -FROM ${REPOSITORY}/mucoll-minimal:${VERSION}-alma9 +ARG IMAGE=madbaron +FROM ${IMAGE}/mucoll-minimal:${VERSION}-alma9 -ARG MUCOLL_BRANCH=master +ARG MUCOLL_SHA=1c4ba29cebeb3e9744626ffc0d04e6ac3f22265e +ARG REPOSITORY=https://github.com/madbaron/mucoll-spack # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone -b ${MUCOLL_BRANCH} --single-branch https://github.com/madbaron/mucoll-spack ${REPOPATH} + git clone ${REPOSITORY} ${REPOPATH} && \ + cd ${REPOPATH} && \ + git checkout ${MUCOLL_SHA} && \ + cd - # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh From c31f482d62998ae2dbf2d829fba6f7ec29a88722 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 22:02:58 +0100 Subject: [PATCH 70/80] fix for repo name --- AlmaLinux9/Dockerfile-sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index 5e01c21..b3d0bd1 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -13,7 +13,7 @@ ARG REPOSITORY=https://github.com/madbaron/mucoll-spack # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone ${REPOSITORY} ${REPOPATH} && \ + git clone https://github.com/${REPOSITORY} ${REPOPATH} && \ cd ${REPOPATH} && \ git checkout ${MUCOLL_SHA} && \ cd - From 9f1ccdc4a36f8d9bdd03018696cf5bcdfa7a25d6 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Mon, 25 Nov 2024 22:08:03 +0100 Subject: [PATCH 71/80] cleanup of images --- .github/workflows/full-rebuild.yml | 5 +++-- .github/workflows/minimal-rebuild.yml | 5 +++-- AlmaLinux9/Dockerfile-sim | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index b762341..abe5f89 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -167,6 +167,7 @@ jobs: file: ./${{matrix.os.dir}}/Dockerfile-sim build-args: | VERSION=${{ steps.meta.outputs.version }} - REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} - MUCOLL_BRANCH=${{ steps.meta.outputs.version }} + IMAGE=${{ env.REGISTRY }}/${{env.OWNER_LC}} + REPOSITORY=${{ github.repository }} + MUCOLL_SHA=${{ github.sha }} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} \ No newline at end of file diff --git a/.github/workflows/minimal-rebuild.yml b/.github/workflows/minimal-rebuild.yml index 91bc0fe..3d81372 100644 --- a/.github/workflows/minimal-rebuild.yml +++ b/.github/workflows/minimal-rebuild.yml @@ -121,6 +121,7 @@ jobs: file: ./${{matrix.os.dir}}/Dockerfile-sim build-args: | VERSION=${{ steps.meta.outputs.version }} - REPOSITORY=${{ env.REGISTRY }}/${{env.OWNER_LC}} - MUCOLL_BRANCH=${{ steps.meta.outputs.version }} + IMAGE=${{ env.REGISTRY }}/${{env.OWNER_LC}} + REPOSITORY=${{ github.repository }} + MUCOLL_SHA=${{ github.sha }} tags: ${{ env.REGISTRY }}/${{env.OWNER_LC}}/mucoll-sim:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} \ No newline at end of file diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index b3d0bd1..a515590 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -8,7 +8,7 @@ ARG IMAGE=madbaron FROM ${IMAGE}/mucoll-minimal:${VERSION}-alma9 ARG MUCOLL_SHA=1c4ba29cebeb3e9744626ffc0d04e6ac3f22265e -ARG REPOSITORY=https://github.com/madbaron/mucoll-spack +ARG REPOSITORY=madbaron/mucoll-spack # Adding repository: MuColl RUN source /opt/setup_spack.sh && \ From 3b80f7d93c834ab0b53e273da0645719212beeb5 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Tue, 26 Nov 2024 09:22:52 +0100 Subject: [PATCH 72/80] added back concretization workflow to run on PRs --- .github/workflows/concretize-template.yaml | 48 ++++++++++++++++++++++ .github/workflows/concretize.yaml | 13 ++++++ 2 files changed, 61 insertions(+) create mode 100644 .github/workflows/concretize-template.yaml create mode 100644 .github/workflows/concretize.yaml diff --git a/.github/workflows/concretize-template.yaml b/.github/workflows/concretize-template.yaml new file mode 100644 index 0000000..57feb02 --- /dev/null +++ b/.github/workflows/concretize-template.yaml @@ -0,0 +1,48 @@ +name: concretize-template + +on: + workflow_call: + inputs: + target: + required: true + type: string +jobs: + concretize: + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - name: Checkout Spack + uses: actions/checkout@v4 + with: + repository: spack/spack + path: spack + + - name: Checkout Key4hep + uses: actions/checkout@v4 + with: + repository: key4hep/key4hep-spack + path: key4hep-spack + + - name: Checkout this repo + uses: actions/checkout@v4 + with: + path: mucoll-spack + + - name: Concretize mucoll-common + run: | + source spack/share/spack/setup-env.sh + spack --version + spack repo add key4hep-spack + spack repo add mucoll-spack + spack compiler find + spack env activate mucoll-spack/environments/mucoll-${{ inputs.target }} + spack add mucoll-stack %gcc@11: + spack concretize + spack spec -Nt > ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log + + - name: Store artifacts + uses: actions/upload-artifact@v4 + with: + name: spec-list + path: ${GITHUB_WORKSPACE}/spec-${{ inputs.target }}.log \ No newline at end of file diff --git a/.github/workflows/concretize.yaml b/.github/workflows/concretize.yaml new file mode 100644 index 0000000..1e3fb1f --- /dev/null +++ b/.github/workflows/concretize.yaml @@ -0,0 +1,13 @@ +name: concretize-ubuntu + +on: [pull_request] + +jobs: + concretize: + strategy: + fail-fast: false + matrix: + target: [common, release, release-debug] + uses: ./.github/workflows/concretize-template.yaml + with: + target: ${{ matrix.target }} \ No newline at end of file From 555ae527adecca46c3ca7cd91c1b35c842f547d7 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Tue, 26 Nov 2024 09:28:46 +0100 Subject: [PATCH 73/80] removed full rebuild from PR --- .github/workflows/full-rebuild.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/full-rebuild.yml b/.github/workflows/full-rebuild.yml index abe5f89..8fcc226 100644 --- a/.github/workflows/full-rebuild.yml +++ b/.github/workflows/full-rebuild.yml @@ -4,10 +4,6 @@ on: paths: - .github/workflows/full-rebuild.yml - 'AlmaLinux9/Dockerfile-spack' - pull_request: - branches: - - 'master' - - 'main' workflow_dispatch: env: From ce95a50b16b2edfd0321f1da8354f5f1a2fffda5 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Tue, 26 Nov 2024 11:45:47 +0100 Subject: [PATCH 74/80] tentative cleanup of paths --- packages/mucoll-stack/package.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/packages/mucoll-stack/package.py b/packages/mucoll-stack/package.py index 5785244..48e9952 100644 --- a/packages/mucoll-stack/package.py +++ b/packages/mucoll-stack/package.py @@ -151,16 +151,9 @@ def setup_run_environment(self, env): env.set('MUCOLL_GEO', os.path.join(self.spec['lcgeo'].prefix.share.lcgeo.compact, 'MuColl/MuColl_v1/MuColl_v1.xml')) env.set('MUCOLL_RELEASE_VERSION', self.spec.version) - env.prepend_path('LD_LIBRARY_PATH', self.spec['root'].prefix.lib.root) - - # remove when https://github.com/spack/spack/pull/37881 is merged - env.prepend_path('LD_LIBRARY_PATH', self.spec['podio'].libs.directories[0]) - env.prepend_path('LD_LIBRARY_PATH', self.spec['edm4hep'].libs.directories[0]) - env.prepend_path('LD_LIBRARY_PATH', self.spec['lcio'].libs.directories[0]) - - # remove when https://github.com/spack/spack/pull/38015 is merged - env.prepend_path('LD_LIBRARY_PATH', self.spec['dd4hep'].prefix.lib) - env.prepend_path('LD_LIBRARY_PATH', self.spec['dd4hep'].prefix.lib64) + # ROOT needs to be in LD_LIBRARY_PATH to prevent using system installations + env.prepend_path("LD_LIBRARY_PATH", self.spec["root"].prefix.lib) + env.prepend_path("PYTHONPATH", self.spec["root"].prefix.lib) def install(self, spec, prefix): return install_setup_script(self, spec, prefix, 'MUCOLL_LATEST_SETUP_PATH') From 7a64454a1717c75a8b278eafd57e099845fd3705 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Tue, 26 Nov 2024 15:57:52 +0100 Subject: [PATCH 75/80] requing LCIO support for dd4hep --- environments/mucoll-release/spack.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index 685dcb8..b0fecb0 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -18,6 +18,8 @@ spack: geant4: require: +opengl+qt~vecgeom cxxstd=20 + dd4hep: + require: +edm4hep+lcio+xercesc+hepmc3 overlay: require: '@0.25' @@ -89,5 +91,7 @@ spack: concretizer: unify: true + view: false + specs: - mucoll-stack \ No newline at end of file From 81a7e5102826ed584986b287a5cb2cdd037c3081 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Wed, 27 Nov 2024 11:21:47 +0100 Subject: [PATCH 76/80] turning off view without changing the setup script wasn't smart --- environments/mucoll-release/spack.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index b0fecb0..c70f6a9 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -91,7 +91,5 @@ spack: concretizer: unify: true - view: false - specs: - mucoll-stack \ No newline at end of file From 1db732d59dab89a5f0a1c20f8dad8e55210d0347 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Wed, 27 Nov 2024 14:43:38 +0100 Subject: [PATCH 77/80] picking up marlintrackprocessors master --- environments/mucoll-release/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index c70f6a9..f7079ba 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -32,7 +32,7 @@ spack: conformaltracking: require: '@1.12.1' marlintrkprocessors: - require: '@2.15.2' + require: '@master' actstracking: require: '@1.2.2' muoncvxddigitiser: From b81145f46eb6b496618a8c14325f01d1072123b2 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 29 Nov 2024 15:26:09 +0100 Subject: [PATCH 78/80] updating ACTSTracking package --- packages/actstracking/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/actstracking/package.py b/packages/actstracking/package.py index d5bdb34..3a7d349 100644 --- a/packages/actstracking/package.py +++ b/packages/actstracking/package.py @@ -16,7 +16,8 @@ class Actstracking(CMakePackage, MCIlcsoftpackage): maintainers = ['gianelle', 'kkrizka'] version('main', branch='main') - version('1.2.2', sha256='be08b87037167892a9b1a7ad601511beaf99423e836841436c6318fef5fa93de', preferred=True) + version('1.3.0', sha256='c14d552342d7c7f26ed5441cd7008263a6f4772f', preferred=True) + version('1.2.2', sha256='be08b87037167892a9b1a7ad601511beaf99423e836841436c6318fef5fa93de') version('1.2.1', sha256='747c15a4c937ab09d79afcc956bb1f1f82ce345febfb4bd18462b71e70ae0b29') version('1.2', sha256='7390d03ab848f7ad9e67c5aabda8122942a885256775174db30964fb9fe028e1') version('1.1.0', sha256='d565e70a2fec97d0d2e81ada69ed54ef8dacc44b0f608b4cf3dffa561091afeb') @@ -39,6 +40,10 @@ def setup_run_environment(self, spack_env): spack_env.prepend_path('MARLIN_DLL', self.prefix.lib + "/libACTSTracking.so") spack_env.set("ACTS_TGeoFile", self.prefix.share.ACTSTracking.data + "/MuColl_v1.root") spack_env.set("ACTS_MatFile", self.prefix.share.ACTSTracking.data + "/material-maps.json") + spack_env.set("ACTS_TGeoFile_MuSIC", self.prefix.share.ACTSTracking.data + "/MuSIC_v2.root") + spack_env.set("ACTS_MatFile_MuSIC", self.prefix.share.ACTSTracking.data + "/material-maps.json") + spack_env.set("ACTS_TGeoFile_MAIA", self.prefix.share.ACTSTracking.data + "/MAIA_v0.root") + spack_env.set("ACTS_MatFile_MAIA", self.prefix.share.ACTSTracking.data + "/MAIA_v0_material.json") def cmake_args(self): # C++ Standard From ae2691c06d88cb38d1c480b8d30da9fad0c50902 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 29 Nov 2024 15:32:56 +0100 Subject: [PATCH 79/80] building with new actstracking --- environments/mucoll-release/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environments/mucoll-release/spack.yaml b/environments/mucoll-release/spack.yaml index f7079ba..ee02501 100644 --- a/environments/mucoll-release/spack.yaml +++ b/environments/mucoll-release/spack.yaml @@ -34,7 +34,7 @@ spack: marlintrkprocessors: require: '@master' actstracking: - require: '@1.2.2' + require: '@1.3.0' muoncvxddigitiser: require: '@master' From 9cfa2fa5c2f74219eef37045832cf026f84b9ba1 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Fri, 29 Nov 2024 20:28:53 +0100 Subject: [PATCH 80/80] fixing checksum --- packages/actstracking/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/actstracking/package.py b/packages/actstracking/package.py index 3a7d349..887cd15 100644 --- a/packages/actstracking/package.py +++ b/packages/actstracking/package.py @@ -16,7 +16,7 @@ class Actstracking(CMakePackage, MCIlcsoftpackage): maintainers = ['gianelle', 'kkrizka'] version('main', branch='main') - version('1.3.0', sha256='c14d552342d7c7f26ed5441cd7008263a6f4772f', preferred=True) + version('1.3.0', sha256='d013a7700ce453054848572603bcfc6fdf4f5a4d', preferred=True) version('1.2.2', sha256='be08b87037167892a9b1a7ad601511beaf99423e836841436c6318fef5fa93de') version('1.2.1', sha256='747c15a4c937ab09d79afcc956bb1f1f82ce345febfb4bd18462b71e70ae0b29') version('1.2', sha256='7390d03ab848f7ad9e67c5aabda8122942a885256775174db30964fb9fe028e1')