From 0450997697be6c1011ccdc4820b5c407596fef48 Mon Sep 17 00:00:00 2001 From: Federico Meloni Date: Wed, 20 Nov 2024 14:59:27 +0100 Subject: [PATCH] testing split build --- .github/workflows/build-image.yml | 100 +++++++++++++++++++++++++++--- AlmaLinux9/Dockerfile-base | 59 ++++++++++++++++++ AlmaLinux9/Dockerfile-sim | 26 +++----- AlmaLinux9/Dockerfile-spack | 1 - 4 files changed, 157 insertions(+), 29 deletions(-) create mode 100644 AlmaLinux9/Dockerfile-base diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 5fab43d..4835d43 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -1,25 +1,36 @@ name: build-image on: + push: + branches: + - 'master' + - 'main' + - 'key4hep*' + tags: + - 'v*' + pull_request: + branches: + - 'master' + - 'main' workflow_dispatch: jobs: - build: + build-spack: runs-on: ubuntu-latest strategy: matrix: os: [{dir: AlmaLinux9,suffix: alma9}] steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + password: ${{ secrets.GHCR_TOKEN }} - name: lowercase owner name run: | echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} @@ -30,9 +41,9 @@ jobs: uses: docker/metadata-action@v4 with: images: ghcr.io/${{env.OWNER_LC}}/mucoll-dummy - - name: Build and Push (mucoll-spack) - id: docker_build_mucoll-spack - uses: docker/build-push-action@v4 + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 with: push: true context: ./${{matrix.os.dir}} @@ -41,9 +52,78 @@ jobs: VERSION=${{ steps.meta.outputs.version }} REPOSITORY=ghcr.io/${{env.OWNER_LC}} tags: ghcr.io/${{env.OWNER_LC}}/mucoll-spack:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} - - name: Build and Push (mucoll-sim) - id: docker_build_mucoll-sim - uses: docker/build-push-action@v4 + build-base: + runs-on: ubuntu-latest + needs: build-spack + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 + with: + push: true + context: ./${{matrix.os.dir}} + file: ./${{matrix.os.dir}}/Dockerfile-base + build-args: | + VERSION=${{ steps.meta.outputs.version }} + REPOSITORY=ghcr.io/${{env.OWNER_LC}} + tags: ghcr.io/${{env.OWNER_LC}}/mucoll-base:${{steps.meta.outputs.version}}-${{matrix.os.suffix}} + build-sim: + runs-on: ubuntu-latest + needs: build-base + strategy: + matrix: + os: [{dir: AlmaLinux9,suffix: alma9}] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GHCR_TOKEN }} + - name: lowercase owner name + run: | + echo "OWNER_LC=${OWNER,,}" >>${GITHUB_ENV} + env: + OWNER: '${{ github.repository_owner }}' + - name: Docker Meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{env.OWNER_LC}}/mucoll-dummy + - name: Build and Push + id: docker_build + uses: docker/build-push-action@v6 with: push: true context: ./${{matrix.os.dir}} diff --git a/AlmaLinux9/Dockerfile-base b/AlmaLinux9/Dockerfile-base new file mode 100644 index 0000000..3d2f544 --- /dev/null +++ b/AlmaLinux9/Dockerfile-base @@ -0,0 +1,59 @@ +############################################################################### +# Repository: ${REPOSITORY}/key4hep-base +# Tag: ${VERSION}-alma9 +############################################################################### + +ARG VERSION=main +ARG REPOSITORY=madbaron +FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 + +# Adding Key4hep repo +ARG KEY4HEP_COMMIT=9f4187ee47dfa1828be936e34c23035818bcd3aa + +RUN source /opt/setup_spack.sh && \ + REPOPATH=${SPACK_ROOT}/var/key4hep-spack && \ + git clone -b base-environment --single-branch https://github.com/key4hep/key4hep-spack ${REPOPATH} && \ + if [ -n "${KEY4HEP_COMMIT}" ]; then \ + cd ${REPOPATH}; \ + git checkout ${KEY4HEP_COMMIT}; \ + fi + +# Apply our patches to spack +COPY apply_patches.sh ./apply_patches.sh + +RUN source /opt/setup_spack.sh && \ + ./apply_patches.sh ${SPACK_ROOT}/var/key4hep-spack + +# Add the package repositories +RUN source /opt/setup_spack.sh && \ + spack repo add --scope system ${SPACK_ROOT}/var/key4hep-spack + +# Create the release environment +RUN source /opt/setup_spack.sh && \ + cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-base && \ + spack env activate . && \ + cd - && \ + echo "source /opt/setup_spack.sh" > ${HOME}/setup_env.sh && \ + echo "cd ${SPACK_ROOT}/var/key4hep-spack/environments/key4hep-dev-base" >> ${HOME}/setup_env.sh && \ + echo "spack env activate ." >> ${HOME}/setup_env.sh && \ + echo "cd -" >> ${HOME}/setup_env.sh && \ + echo "spack env status" >> ${HOME}/setup_env.sh + +# Concretizing the stack reusing system packages as external +RUN source ${HOME}/setup_env.sh && \ + spack add key4hep-base-stack && \ + spack concretize --reuse + +# Installing fragments of dependency tree in separate layers for cached debugging +ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast" + +RUN source ${HOME}/setup_env.sh && \ + spack spec -NIt && \ + spack install ${SPACK_INSTALL_OPTS} && \ + spack clean -a + +RUN source ${HOME}/setup_env.sh && \ + echo "source ${SPACK_ENV}/.spack-env/view/setup.sh" > /opt/setup_k4base.sh && \ + echo "alias setup_k4base=\"source /opt/setup_k4base.sh\"" >> /etc/profile.d/aliases.sh + + diff --git a/AlmaLinux9/Dockerfile-sim b/AlmaLinux9/Dockerfile-sim index ee7fb9a..8fd2f1e 100644 --- a/AlmaLinux9/Dockerfile-sim +++ b/AlmaLinux9/Dockerfile-sim @@ -3,25 +3,16 @@ # Tag: ${VERSION}-alma9 ############################################################################### -ARG VERSION=release -ARG REPOSITORY=infnpd -FROM ${REPOSITORY}/mucoll-spack:${VERSION}-alma9 +ARG VERSION=dev +ARG REPOSITORY=madbaron +FROM ${REPOSITORY}/mucoll-base:${VERSION}-alma9 -# Adding repositories: Key4hep + MuColl -ARG KEY4HEP_COMMIT=f6b94a45e2c9e3e414d75606927a56da59c4f00a -ARG MUCOLL_COMMIT=3ebc9b2b984fe33590b768489a7bf54050799407 - -RUN source /opt/setup_spack.sh && \ - REPOPATH=${SPACK_ROOT}/var/key4hep-spack && \ - git clone https://github.com/key4hep/key4hep-spack ${REPOPATH} && \ - if [ -n "${KEY4HEP_COMMIT}" ]; then \ - cd ${REPOPATH}; \ - git checkout ${KEY4HEP_COMMIT}; \ - fi +# Adding repository: MuColl +ARG MUCOLL_COMMIT=3ff04277d2743a36bc4e6b6d162af22e3d2e4bd7 RUN source /opt/setup_spack.sh && \ REPOPATH=${SPACK_ROOT}/var/mucoll-spack && \ - git clone https://github.com/MuonColliderSoft/mucoll-spack ${REPOPATH} && \ + git clone https://github.com/madbaron/mucoll-spack ${REPOPATH} && \ if [ -n "${MUCOLL_COMMIT}" ]; then \ cd ${REPOPATH}; \ git checkout ${MUCOLL_COMMIT}; \ @@ -30,7 +21,6 @@ RUN source /opt/setup_spack.sh && \ # Apply our patches to spack COPY apply_patches.sh ./apply_patches.sh RUN source /opt/setup_spack.sh && \ - ./apply_patches.sh ${SPACK_ROOT}/var/key4hep-spack && \ ./apply_patches.sh ${SPACK_ROOT}/var/mucoll-spack # Add the package repositories @@ -50,10 +40,10 @@ RUN source /opt/setup_spack.sh && \ # Concretizing the MuColl stack reusing system packages as external RUN source ${HOME}/setup_env.sh && \ spack add mucoll-stack && \ - spack concretize --reuse --deprecated + spack concretize --reuse # Installing fragments of dependency tree in separate layers for cached debugging -ENV SPACK_INSTALL_OPTS "--only-concrete --no-add --fail-fast --deprecated" +ENV SPACK_INSTALL_OPTS="--only-concrete --no-add --fail-fast --deprecated" RUN source ${HOME}/setup_env.sh && \ spack spec -NIt && \ diff --git a/AlmaLinux9/Dockerfile-spack b/AlmaLinux9/Dockerfile-spack index d0ec153..fdb7988 100644 --- a/AlmaLinux9/Dockerfile-spack +++ b/AlmaLinux9/Dockerfile-spack @@ -4,7 +4,6 @@ ############################################################################### ARG VERSION=release -ARG REPOSITORY=infnpd FROM gitlab-registry.cern.ch/key4hep/k4-deploy/alma9-build # Add the new Certificate Authority (needed for xrootd)