From 8e6c33dc6f9494d790097d1740ff83d39a3753af Mon Sep 17 00:00:00 2001 From: Oksana Shadura Date: Mon, 20 Nov 2023 16:20:13 +0100 Subject: [PATCH] Simplify GH actions: only 2 images for now both using matrix build --- .github/workflows/dockerci-dev.yml | 319 ++--------------------------- 1 file changed, 21 insertions(+), 298 deletions(-) diff --git a/.github/workflows/dockerci-dev.yml b/.github/workflows/dockerci-dev.yml index c1750934..4dc61d63 100644 --- a/.github/workflows/dockerci-dev.yml +++ b/.github/workflows/dockerci-dev.yml @@ -11,86 +11,28 @@ on: env: PROJECT: 'coffea-casa' GITHUB_ACTIONS: 'true' - REGISTRY: 'localhost:5000' - REGISTRY_PATH: ${{ github.workspace }}/registry - CACHE_PATH: /tmp/.buildx-cache + #REGISTRY: 'localhost:5000' + #REGISTRY_PATH: ${{ github.workspace }}/registry + #CACHE_PATH: /tmp/.buildx-cache jobs: - version-check: + matrix-build: + strategy: + fail-fast: false + matrix: + image: [cc-base-ubuntu, cc-analysis-ubuntu] + # Keep this line in sync with gh actions @ coffea-dask repo + # python: [3.8, 3.9, '3.10'] + #exclude: + # - image: cc7 + # python: 3.8 + name: ${{ matrix.image }} runs-on: ubuntu-latest steps: - name: Fetch current date (with timestamp) id: date run: | echo "::set-output name=date::$(date +"%Y-%m-%d_%H-%M-%S")" - outputs: - date: ${{ steps.date.outputs.date }} - - dask-base: - runs-on: ubuntu-latest - needs: version-check - steps: - - name: Check out code - uses: actions/checkout@v3 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - version: latest - driver-opts: network=host - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: ${{ env.CACHE_PATH }} - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to Harbor Hub - if: success() && github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - registry: hub.opensciencegrid.org - username: ${{ secrets.HARBOR_USER }} - password: ${{ secrets.HARBOR_PASSWORD }} - - name: Start Docker registry - run: | - docker run --rm --detach --publish 5000:5000 \ - --volume ${{ env.REGISTRY_PATH }}:/var/lib/registry \ - --name registry registry:2 - sleep 10 - - name: Build - uses: docker/build-push-action@v3.0.0 - with: - #outputs: type=docker,dest=/tmp/cc-base-ubuntu.tar - #push: ${{ github.event_name != 'pull_request' }} - push: true - cache-from: | - type=local,src=${{ env.CACHE_PATH }} - cache-to: type=local,dest=${{ env.CACHE_PATH }} - builder: ${{ steps.buildx.outputs.name }} - context: docker/ - file: docker/Dockerfile.cc-base-ubuntu - tags: | - localhost:5000/coffea-casa/cc-base-ubuntu:${{ needs.version-check.outputs.date }} - build-args: | - TAG=${{ needs.version-check.outputs.date }} - PROJECT=${{ env.PROJECT }} - GITHUB_ACTIONS=${{ env.GITHUB_ACTIONS }} - REGISTRY=${{ env.REGISTRY }} - - name: Upload Docker registry data for testing - uses: actions/upload-artifact@v2 - with: - name: docker-registry-data - path: ${{ env.REGISTRY_PATH }}/ - retention-days: 1 - - dask-cc: - runs-on: ubuntu-latest - needs: [version-check, dask-base] - steps: - name: Check out code uses: actions/checkout@v2 - name: Set up QEMU @@ -101,14 +43,6 @@ jobs: with: version: latest driver-opts: network=host - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: ${{ env.CACHE_PATH }} - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - name: Login to Harbor Hub if: success() && github.event_name != 'pull_request' uses: docker/login-action@v2 @@ -116,227 +50,16 @@ jobs: registry: hub.opensciencegrid.org username: ${{ secrets.HARBOR_USER }} password: ${{ secrets.HARBOR_PASSWORD }} - - name: Download Docker registry data from build job - uses: actions/download-artifact@v2 - with: - name: docker-registry-data - path: ${{ env.REGISTRY_PATH }} - - name: Start Docker registry - run: | - docker run --rm --detach --publish 5000:5000 \ - --volume ${{ env.REGISTRY_PATH }}:/var/lib/registry \ - --name registry registry:2 - sleep 10 - - name: Import Docker images - run: docker pull localhost:5000/coffea-casa/cc-base-ubuntu:${{ needs.version-check.outputs.date }} - - name: Build + - name: Build and push uses: docker/build-push-action@v3.0.0 with: - #push: ${{ github.event_name != 'pull_request' }} - push: true - cache-from: | - type=local,src=${{ env.CACHE_PATH }} - cache-to: type=local,dest=${{ env.CACHE_PATH }} context: docker/ - file: docker/Dockerfile.cc-ubuntu - tags: | - localhost:5000/coffea-casa/cc-ubuntu:${{ needs.version-check.outputs.date }} - build-args: | + file: docker/Dockerfile.${{ matrix.image }} + push: ${{ github.event_name == 'push' }} + platforms: linux/amd64 + tags: coffea-casa/${{ matrix.image }}:${{ steps.date.outputs.date }} + build-args: DEV=true - TAG=${{ needs.version-check.outputs.date }} + TAG=${{ steps.date.outputs.date }} PROJECT=${{ env.PROJECT }} GITHUB_ACTIONS=${{ env.GITHUB_ACTIONS }} - REGISTRY=${{ env.REGISTRY }} - - name: Upload Docker registry data for testing - uses: actions/upload-artifact@v2 - with: - name: docker-registry-data - path: ${{ env.REGISTRY_PATH }}/ - retention-days: 1 - -# dask-skyhook: -# runs-on: ubuntu-latest -# needs: [version-check, dask-cc] -# steps: -# - name: Check out code -# uses: actions/checkout@v2 -# - name: Set up QEMU -# uses: docker/setup-qemu-action@v2 -# - name: Set up Docker Buildx -# id: buildx -# uses: docker/setup-buildx-action@v2 -# with: -# version: latest -# driver-opts: network=host -# - name: Cache Docker layers -# uses: actions/cache@v2 -# id: cache -# with: -# path: ${{ env.CACHE_PATH }} -# key: ${{ runner.os }}-buildx-${{ github.sha }} -# restore-keys: | -# ${{ runner.os }}-buildx- -# - name: Login to Harbor Hub -# if: success() && github.event_name != 'pull_request' -# uses: docker/login-action@v2 -# with: -# registry: hub.opensciencegrid.org -# username: ${{ secrets.HARBOR_USER }} -# password: ${{ secrets.HARBOR_PASSWORD }} -# - name: Download Docker registry data from build job -# uses: actions/download-artifact@v2 -# with: -# name: docker-registry-data -# path: ${{ env.REGISTRY_PATH }} -# - name: Start Docker registry -# run: | -# docker run --rm --detach --publish 5000:5000 \ -# --volume ${{ env.REGISTRY_PATH }}:/var/lib/registry \ -# --name registry registry:2 -# sleep 10 -# - name: Import Docker images -# run: docker pull localhost:5000/coffea-casa/cc-ubuntu:${{ needs.version-check.outputs.date }} -# - name: Build -# uses: docker/docker/build-push-action@v3.0.0 -# with: -# push: ${{ github.event_name != 'pull_request' }} -# cache-from: | -# type=local,src=${{ env.CACHE_PATH }} -# cache-to: type=local,dest=${{ env.CACHE_PATH }} -# context: docker/ -# file: docker/Dockerfile.cc-base-centos7 -# tags: | -# localhost:5000/coffea-casa/cc-ubuntu-skyhook:${{ needs.version-check.outputs.date }} -# build-args: | -# TAG=${{ needs.version-check.outputs.date }} -# PROJECT=${{ env.PROJECT }} -# GITHUB_ACTIONS=${{ env.GITHUB_ACTIONS }} -# REGISTRY=${{ env.REGISTRY }} -# - name: Cleanup -# run: | -# docker stop registry - - dask-analysis: - runs-on: ubuntu-latest - needs: version-check - steps: - - name: Check out code - uses: actions/checkout@v2 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - with: - version: latest - driver-opts: network=host - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: ${{ env.CACHE_PATH }} - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- - - name: Login to Harbor Hub - if: success() && github.event_name != 'pull_request' - uses: docker/login-action@v2 - with: - registry: hub.opensciencegrid.org - username: ${{ secrets.HARBOR_USER }} - password: ${{ secrets.HARBOR_PASSWORD }} - - name: Start Docker registry - run: | - docker run --rm --detach --publish 5000:5000 \ - --volume ${{ env.REGISTRY_PATH }}:/var/lib/registry \ - --name registry registry:2 - sleep 10 - - name: Build - uses: docker/build-push-action@v2 - with: - #push: ${{ github.event_name != 'pull_request' }} - push: true - #outputs: type=docker,dest=/tmp/cc-analysis-ubuntu.tar - cache-from: | - type=local,src=${{ env.CACHE_PATH }} - cache-to: type=local,dest=${{ env.CACHE_PATH }} - context: docker/ - file: docker/Dockerfile.cc-analysis-ubuntu - tags: | - localhost:5000/coffea-casa/cc-analysis-ubuntu:${{ needs.version-check.outputs.date }} - build-args: | - DEV=true - TAG=${{ needs.version-check.outputs.date }} - PROJECT=${{ env.PROJECT }} - GITHUB_ACTIONS=${{ env.GITHUB_ACTIONS }} - REGISTRY=${{ env.REGISTRY }} - - name: Upload Docker registry data for testing - uses: actions/upload-artifact@v2 - with: - name: docker-registry-data - path: ${{ env.REGISTRY_PATH }}/ - retention-days: 1 - - -# dask-analysis-skyhook: -# runs-on: ubuntu-latest -# needs: [version-check, dask-analysis] -# steps: -# - name: Check out code -# uses: actions/checkout@v2 -# - name: Set up QEMU -# uses: docker/setup-qemu-action@v2 -# - name: Set up Docker Buildx -# id: buildx -# uses: docker/setup-buildx-action@v2 -# with: -# version: latest -# driver-opts: network=host -# - name: Cache Docker layers -# uses: actions/cache@v2 -# id: cache -# with: -# path: ${{ env.CACHE_PATH }} -# key: ${{ runner.os }}-buildx-${{ github.sha }} -# restore-keys: | -# ${{ runner.os }}-buildx- -# - name: Login to Harbor Hub -# if: success() && github.event_name != 'pull_request' -# uses: docker/login-action@v2 -# with: -# registry: hub.opensciencegrid.org -# username: ${{ secrets.HARBOR_USER }} -# password: ${{ secrets.HARBOR_PASSWORD }} -# - name: Download Docker registry data from build job -# uses: actions/download-artifact@v2 -# with: -# name: docker-registry-data -# path: ${{ env.REGISTRY_PATH }} -# - name: Start Docker registry -# run: | -# docker run --rm --detach --publish 5000:5000 \ -# --volume ${{ env.REGISTRY_PATH }}:/var/lib/registry \ -# --name registry registry:2 -# sleep 10 -# - name: Import Docker images -# run: docker pull localhost:5000/coffea-casa/cc-analysis-ubuntu:${{ needs.version-check.outputs.date }} -# - name: Build -# uses: docker/build-push-action@v3.0.0 -# with: -# #push: ${{ github.event_name != 'pull_request' }} -# push: true -# cache-from: | -# type=local,src=${{ env.CACHE_PATH }} -# cache-to: type=local,dest=${{ env.CACHE_PATH }} -# context: docker/ -# file: docker/Dockerfile.cc-analysis-ubuntu-skyhook -# tags: | -# localhost:5000/coffea-casa/cc-analysis-ubuntu-skyhook:${{ needs.version-check.outputs.date }} -# build-args: | -# TAG=${{ needs.version-check.outputs.date }} -# PROJECT=${{ env.PROJECT }} -# GITHUB_ACTIONS=${{ env.GITHUB_ACTIONS }} -# REGISTRY=${{ env.REGISTRY }} -# - name: Cleanup -# run: | -# docker stop registry