From 4d290a78b4e0820653608e6aa47126818be5a67a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 9 Jan 2025 16:02:35 -0600 Subject: [PATCH 01/73] Initial implementation --- action.yaml | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 action.yaml diff --git a/action.yaml b/action.yaml new file mode 100644 index 0000000..fab1867 --- /dev/null +++ b/action.yaml @@ -0,0 +1,141 @@ +--- +inputs: + image-repository: + required: true + context: + default: "." + build-args: + required: false + build-secrets: + required: false + from-scratch: + description: Do not read from the cache when building the image + default: "false" +outputs: + image: + value: ${{ inputs.image-repository }}@${{ steps.build-push.outputs.digest }} + image-repository: + value: ${{ inputs.image-repository }} + digest: + value: ${{ steps.build-push.outputs.digest }} + tags: + value: ${{ steps.tags.outputs.json }} + commit-sha: + value: ${{ steps.commit-sha.outputs.head }} +runs: + using: composite + steps: + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + - name: Determine commit SHA + id: commit-sha + shell: bash + run: | + # Determine commit SHA + head="$(git rev-parse HEAD)" + echo "head=$head" | tee -a "$GITHUB_OUTPUT" + # Optional branch name (e.g. "main") for workflows triggered by `pull_request` or `push` events. + - name: Branch + id: branch + shell: bash + run: | + # Branch + echo "name=${branch}" | tee -a "$GITHUB_OUTPUT" + env: + branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }} + - name: Docker metadata + id: metadata + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: | + ${{ inputs.image-repository }} + tags: | + type=sha,prefix=sha- + type=ref,prefix=pr-,event=pr + type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: "true" + # Use separate cache images to avoid bloating final images + # https://docs.docker.com/build/cache/backends/registry/ + - name: Docker cache-from + id: cache-from + if: ${{ inputs.from-scratch != 'true' }} + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: | + ${{ inputs.image-repository }} + tags: | + type=sha,prefix=cache-sha-,format=long + type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: "true" + - name: Docker cache-to + id: cache-to + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + with: + images: | + ${{ inputs.image-repository }} + tags: | + type=sha,prefix=cache-sha-,format=long + type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: "true" + - name: Docker cache metadata + id: cache + shell: bash + run: | + # Docker cache metadata + # Specify our multiline output using GH action flavored heredocs + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + { + echo "from-tags< Date: Mon, 13 Jan 2025 16:37:31 -0600 Subject: [PATCH 02/73] Update README --- README.md | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 90ef786..c58c74e 100644 --- a/README.md +++ b/README.md @@ -1 +1,57 @@ -# docker-build \ No newline at end of file +# Docker Build + +Build a Docker image while utilizing [layer caching](https://docs.docker.com/build/cache/) +backed from the image repository. [Docker does support using GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) +as a layer cache backend but the default cache limit for a repository is 10 GB which is +quite small for Docker images. + +We recommend utilizing a separate image repositories for deployment and production (e.g.`temporary/my-image` and `permanent/my-image`) to make it easier to separate temporary images from permanent images meant for end users. Promoting temporary images to be permanent can be done with `docker push` or [`regctl image copy --digest-tags`](https://github.com/regclient/regclient/blob/main/docs/regctl.md#registry-commands) if you want the digest to be identical across registries. + +## Example + +```yaml +--- +jobs: + example: + permissions: {} + runs-on: ubuntu-latest + steps: + - name: Build image + uses: beacon-biosignals/docker-build@v1 + with: + image-repository: temporary/my-image + context: . + # Example of passing in Docker `--build-arg` + build-args: | + JULIA_VERSION=1.10 + PYTHON_VERSION=3.10 + # Example of passing in Docker `--secret` + build-secrets: | + github-token=${{ secrets.token || github.token }} + # Build images from scratch on `main`. Ensures system packages have latest security fixes. + from-scratch: ${{ github.ref == 'refs/heads/main' }} +``` + +## Inputs + +| Name | Description | Required | Example | +|:---------------------|:------------|:---------|:--------| +| `image-repository` | The Docker image repository to push the build image and cached layers. | Yes | `temporary/my-image` | +| `context` | The Docker build context directory. Defaults to `.`. | No | `./my-image` | +| `build-args` | List of [build-time variables](https://docs.docker.com/reference/cli/docker/buildx/build/#build-arg). | No | 
HTTP_PROXY=http://10.20.30.2:1234
FTP_PROXY=http://40.50.60.5:4567
| +| `build-secrets` | List of [secrets](https://docs.docker.com/engine/reference/commandline/buildx_build/#secret) to expose to the build. | No | `GIT_AUTH_TOKEN=mytoken` | +| `from-scratch` | Do not use cache when building the image. Defaults to `false`. | No | `false` | + +## Outputs + +| Name | Description | Example | +|:-------------------|:------------|:--------| +| `image` | Reference to the build image including the digest. | `temporary/my-image@sha256:37782d4e1c24d8f12047039a0d3512d1b6059e306a80d5b66a1d9ff60247a8cb` | +| `image-repository` | The Docker image repository where the image was pushed to. | `temporary/my-image` | +| `digest` | The built Docker image digest. | `sha256:37782d4e1c24d8f12047039a0d3512d1b6059e306a80d5b66a1d9ff60247a8cb` | +| `tags` | JSON list of tags associated with the built Docker image. | `branch-main`, `sha-152cb14` | +| `commit-sha` | The Git commit SHA used to build the imag. | `152cb14643b50529b229930d6124e6bbef48668d` | + +## Permissions + +No [job permissions](https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs) are required to run this action. From 47fb3137c652e9f910f972ae47ce18ec767749c6 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 08:42:51 -0600 Subject: [PATCH 03/73] Add GHAs --- .editorconfig | 12 +++++++++ .github/workflows/gha.yaml | 25 +++++++++++++++++ .github/workflows/integration-tests.yaml | 25 +++++++++++++++++ .github/workflows/shell.yaml | 34 ++++++++++++++++++++++++ .github/workflows/yaml.yaml | 18 +++++++++++++ .yamllint.yaml | 8 ++++++ action.yaml | 4 ++- test/Dockerfile | 30 +++++++++++++++++++++ test/docker.cow | 19 +++++++++++++ 9 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 .editorconfig create mode 100644 .github/workflows/gha.yaml create mode 100644 .github/workflows/integration-tests.yaml create mode 100644 .github/workflows/shell.yaml create mode 100644 .github/workflows/yaml.yaml create mode 100644 .yamllint.yaml create mode 100644 test/Dockerfile create mode 100644 test/docker.cow diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..28491b7 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +# https://editorconfig.org/ + +# https://manpages.debian.org/testing/shfmt/shfmt.1.en.html#EXAMPLES +[*.sh] +indent_style = space +indent_size = 4 +shell_variant = bash # --language-variant +binary_next_line = false +switch_case_indent = true # --case-indent +space_redirects = false +keep_padding = false +function_next_line = false # --func-next-line diff --git a/.github/workflows/gha.yaml b/.github/workflows/gha.yaml new file mode 100644 index 0000000..5272099 --- /dev/null +++ b/.github/workflows/gha.yaml @@ -0,0 +1,25 @@ +--- +name: GitHub Actions +on: + pull_request: + paths: + - ".github/workflows/*" + +jobs: + lint: + name: Lint + # These permissions are needed to: + # - Checkout the Git repo (`contents: read`) + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # https://github.com/rhysd/actionlint/blob/v1.7.6/docs/usage.md#use-actionlint-on-github-actions + # https://github.com/rhysd/actionlint/blob/v1.7.6/docs/usage.md#reviewdog + # https://github.com/reviewdog/reviewdog#filter-mode + # No support for non-workflows yet: https://github.com/rhysd/actionlint/issues/46 + - uses: reviewdog/action-actionlint@a1b7ce56be870acfe94b83ce5f6da076aecc6d8c # v1.62.0 + with: + fail_level: error + filter_mode: nofilter # Post results on all results and not just changed files diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml new file mode 100644 index 0000000..5f129d9 --- /dev/null +++ b/.github/workflows/integration-tests.yaml @@ -0,0 +1,25 @@ +--- +name: Integration Tests +on: + pull_request: + paths: + - "action.yaml" + - ".github/workflows/integration-tests.yaml" + +jobs: + test: + name: Test + # These permissions are needed to: + # - Checkout the repo + permissions: + contents: read + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./ + id: build + with: + image-repository: whalesay + context: test + build-args: | + DEBIAN_VERSION=bookworm \ No newline at end of file diff --git a/.github/workflows/shell.yaml b/.github/workflows/shell.yaml new file mode 100644 index 0000000..1319076 --- /dev/null +++ b/.github/workflows/shell.yaml @@ -0,0 +1,34 @@ +--- +name: Shell +on: + pull_request: + paths: + - "**.sh" + - ".github/workflows/*" + +jobs: + lint-format: + name: Lint & Format + # These permissions are needed to: + # - Checkout the Git repo (`contents: read`) + # - Post a comments on PRs: https://github.com/luizm/action-sh-checker#secrets + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Extract workflow shell scripts + id: extract + uses: beacon-biosignals/gha-extract-shell-scripts@v1 + - uses: luizm/action-sh-checker@c6edb3de93e904488b413636d96c6a56e3ad671a # v0.8.0 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sh_checker_comment: true + # Support investigating linting/formatting errors + - uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: workflow-scripts + path: ${{ steps.extract.outputs.output-dir }} diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml new file mode 100644 index 0000000..91d390c --- /dev/null +++ b/.github/workflows/yaml.yaml @@ -0,0 +1,18 @@ +--- +# https://yamllint.readthedocs.io/en/stable/integration.html#integration-with-github-actions +name: YAML +on: + pull_request: + paths: + - "**/*.yaml" + - "**/*.yml" +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install yamllint + run: pip install yamllint + - name: Lint YAML files + run: yamllint . --format=github diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 0000000..da01502 --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,8 @@ +--- +rules: + indentation: + spaces: 2 + indent-sequences: true + document-start: + present: true + new-line-at-end-of-file: enable diff --git a/action.yaml b/action.yaml index fab1867..550a7c5 100644 --- a/action.yaml +++ b/action.yaml @@ -11,6 +11,8 @@ inputs: from-scratch: description: Do not read from the cache when building the image default: "false" + push: + default: "true" outputs: image: value: ${{ inputs.image-repository }}@${{ steps.build-push.outputs.digest }} @@ -121,7 +123,7 @@ runs: build.run-id=${{ github.run_id }} build.run-attempt=${{ github.run_attempt }} build.commit-sha=${{ steps.commit-sha.outputs.head }} - push: true + push: ${{ inputs.push }} provenance: false # Prevent pushing a docker manifest - name: Inspect Docker Manifest shell: bash diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 0000000..173c9c0 --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 + +# Adapted from: +# - https://hub.docker.com/r/docker/whalesay +# - https://github.com/docker/whalesay + +ARG DEBIAN_VERSION +FROM debian:$DEBIAN_VERSION + +# Reduces output from `apt-get` +ENV DEBIAN_FRONTEND="noninteractive" + +# Configure `apt-get` to keep downloaded packages. Needed for using `--mount=type=cache` with `apt-get` +# https://docs.docker.com/engine/reference/python-deps/#example-cache-apt-packages +RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | tee -a /etc/apt/apt.conf.d/keep-cache + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get -qq update && \ + apt-get -qq install cowsay fortune && \ + mv /usr/share/cowsay/cows/default.cow /usr/share/cowsay/cows/cow.cow + +# Add "cowsay" location onto the path +ENV PATH=$PATH:/usr/games + +COPY docker.cow /usr/share/cowsay/cows/ +RUN ln -sf /usr/share/cowsay/cows/docker.cow /usr/share/cowsay/cows/default.cow + +CMD ["/bin/sh", "-c", "fortune | cowsay"] diff --git a/test/docker.cow b/test/docker.cow new file mode 100644 index 0000000..f0bac68 --- /dev/null +++ b/test/docker.cow @@ -0,0 +1,19 @@ +## +## Docker Cow +## +$the_cow = < Date: Tue, 14 Jan 2025 08:49:17 -0600 Subject: [PATCH 04/73] Linting pass --- .github/workflows/integration-tests.yaml | 3 ++- README.md | 6 +++--- action.yaml | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 5f129d9..5e2c5f9 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -22,4 +22,5 @@ jobs: image-repository: whalesay context: test build-args: | - DEBIAN_VERSION=bookworm \ No newline at end of file + DEBIAN_VERSION=bookworm + push: false diff --git a/README.md b/README.md index c58c74e..a7087f8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # Docker Build Build a Docker image while utilizing [layer caching](https://docs.docker.com/build/cache/) -backed from the image repository. [Docker does support using GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) -as a layer cache backend but the default cache limit for a repository is 10 GB which is -quite small for Docker images. +backed from the image repository. Although [Docker does support using GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) +as a layer cache backend but the GHA cache limit for a repository is 10 GB which is +quite limiting for Docker images. We recommend utilizing a separate image repositories for deployment and production (e.g.`temporary/my-image` and `permanent/my-image`) to make it easier to separate temporary images from permanent images meant for end users. Promoting temporary images to be permanent can be done with `docker push` or [`regctl image copy --digest-tags`](https://github.com/regclient/regclient/blob/main/docs/regctl.md#registry-commands) if you want the digest to be identical across registries. diff --git a/action.yaml b/action.yaml index 550a7c5..c12c9d6 100644 --- a/action.yaml +++ b/action.yaml @@ -1,4 +1,9 @@ --- +name: Docker Build +description: Build a Docker image while utilize layer caching from the image repository. +branding: + color: blue + icon: layers inputs: image-repository: required: true From 146460cb28b27af95c44761df12e320f8a5fdd20 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 08:51:07 -0600 Subject: [PATCH 05/73] Skip cache-to when pushing is false --- action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/action.yaml b/action.yaml index c12c9d6..929d7a4 100644 --- a/action.yaml +++ b/action.yaml @@ -81,6 +81,7 @@ runs: DOCKER_METADATA_PR_HEAD_SHA: "true" - name: Docker cache-to id: cache-to + if: ${{ inputs.push == 'true' }} uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: images: | From 7bf6d2913270df6e18b6aed0d8c98446dcf74316 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 08:52:35 -0600 Subject: [PATCH 06/73] Set default for DEBIAN_VERSION --- test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Dockerfile b/test/Dockerfile index 173c9c0..11a317d 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -4,7 +4,7 @@ # - https://hub.docker.com/r/docker/whalesay # - https://github.com/docker/whalesay -ARG DEBIAN_VERSION +ARG DEBIAN_VERSION=bullseye FROM debian:$DEBIAN_VERSION # Reduces output from `apt-get` From 5da6a0b3aefa94c4cdc523ba2e6696a73724805c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:15:22 -0600 Subject: [PATCH 07/73] Try GHCR --- .github/workflows/integration-tests.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 5e2c5f9..529858f 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -16,11 +16,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io/beacon-biosignals/whalesay + username: ${{ github.actor }} + password: ${{ github.token }} - uses: ./ id: build with: - image-repository: whalesay + image-repository: ghcr.io/beacon-biosignals/whalesay context: test build-args: | DEBIAN_VERSION=bookworm - push: false From 57f34879a40be38e0fd3320337cdd01b13436a2d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:17:56 -0600 Subject: [PATCH 08/73] Add permissions --- .github/workflows/integration-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 529858f..438717d 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -13,6 +13,7 @@ jobs: # - Checkout the repo permissions: contents: read + packages: write runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 15ae11dfc9a00477f102fd2020054b3359accf6f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:22:10 -0600 Subject: [PATCH 09/73] Require push --- action.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/action.yaml b/action.yaml index 929d7a4..123b6fc 100644 --- a/action.yaml +++ b/action.yaml @@ -16,8 +16,6 @@ inputs: from-scratch: description: Do not read from the cache when building the image default: "false" - push: - default: "true" outputs: image: value: ${{ inputs.image-repository }}@${{ steps.build-push.outputs.digest }} @@ -34,6 +32,8 @@ runs: steps: - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 + with: + driver: docker-container - name: Determine commit SHA id: commit-sha shell: bash @@ -81,7 +81,6 @@ runs: DOCKER_METADATA_PR_HEAD_SHA: "true" - name: Docker cache-to id: cache-to - if: ${{ inputs.push == 'true' }} uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: images: | @@ -129,7 +128,9 @@ runs: build.run-id=${{ github.run_id }} build.run-attempt=${{ github.run_attempt }} build.commit-sha=${{ steps.commit-sha.outputs.head }} - push: ${{ inputs.push }} + # Required to be true so we can consistently get access to the `digest`: + # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 + push: true provenance: false # Prevent pushing a docker manifest - name: Inspect Docker Manifest shell: bash From 524590b6470a53b921c9d9bf09c447616ec10d7e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:30:05 -0600 Subject: [PATCH 10/73] Overwrite revision annotation --- action.yaml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/action.yaml b/action.yaml index 123b6fc..c0e62df 100644 --- a/action.yaml +++ b/action.yaml @@ -60,6 +60,13 @@ runs: type=sha,prefix=sha- type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys + # TODO: May want to drop `build.commit-sha` + labels: | + org.opencontainers.image.revision=${{ steps.commit-sha.outputs.head }} + build.run-id=${{ github.run_id }} + build.run-attempt=${{ github.run_attempt }} + build.commit-sha=${{ steps.commit-sha.outputs.head }} env: # https://github.com/docker/metadata-action/issues/206 DOCKER_METADATA_PR_HEAD_SHA: "true" @@ -122,12 +129,7 @@ runs: cache-from: ${{ steps.cache.outputs.from-tags }} cache-to: ${{ steps.cache.outputs.to-tags }} tags: ${{ steps.metadata.outputs.tags }} - # TODO: May want to drop `build.commit-sha` - annotations: | - ${{ steps.metadata.outputs.annotations }} - build.run-id=${{ github.run_id }} - build.run-attempt=${{ github.run_attempt }} - build.commit-sha=${{ steps.commit-sha.outputs.head }} + annotations: ${{ steps.metadata.outputs.annotations }} # Required to be true so we can consistently get access to the `digest`: # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 push: true From 21555167f1c19f297f732578b79ffc15eb7d3cdf Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:39:38 -0600 Subject: [PATCH 11/73] Fiddle with annotations --- action.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/action.yaml b/action.yaml index c0e62df..ab36099 100644 --- a/action.yaml +++ b/action.yaml @@ -61,12 +61,8 @@ runs: type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys - # TODO: May want to drop `build.commit-sha` labels: | org.opencontainers.image.revision=${{ steps.commit-sha.outputs.head }} - build.run-id=${{ github.run_id }} - build.run-attempt=${{ github.run_attempt }} - build.commit-sha=${{ steps.commit-sha.outputs.head }} env: # https://github.com/docker/metadata-action/issues/206 DOCKER_METADATA_PR_HEAD_SHA: "true" @@ -129,7 +125,12 @@ runs: cache-from: ${{ steps.cache.outputs.from-tags }} cache-to: ${{ steps.cache.outputs.to-tags }} tags: ${{ steps.metadata.outputs.tags }} - annotations: ${{ steps.metadata.outputs.annotations }} + # TODO: May want to drop `build.commit-sha` + annotations: | + ${{ steps.metadata.outputs.annotations }} + build.run-id=${{ github.run_id }} + build.run-attempt=${{ github.run_attempt }} + build.commit-sha=${{ steps.commit-sha.outputs.head }} # Required to be true so we can consistently get access to the `digest`: # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 push: true From 0a0922047f0d59cbbac440563afab8c3f2d3f224 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:50:05 -0600 Subject: [PATCH 12/73] WIP --- action.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/action.yaml b/action.yaml index ab36099..0be0d3c 100644 --- a/action.yaml +++ b/action.yaml @@ -26,10 +26,11 @@ outputs: tags: value: ${{ steps.tags.outputs.json }} commit-sha: - value: ${{ steps.commit-sha.outputs.head }} + value: ${{ github.event.pull_request.head.sha || github.sha }} runs: using: composite steps: + - run: jq <<<"${{ toJSON(github.event) }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: @@ -60,9 +61,6 @@ runs: type=sha,prefix=sha- type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys - labels: | - org.opencontainers.image.revision=${{ steps.commit-sha.outputs.head }} env: # https://github.com/docker/metadata-action/issues/206 DOCKER_METADATA_PR_HEAD_SHA: "true" @@ -125,12 +123,12 @@ runs: cache-from: ${{ steps.cache.outputs.from-tags }} cache-to: ${{ steps.cache.outputs.to-tags }} tags: ${{ steps.metadata.outputs.tags }} - # TODO: May want to drop `build.commit-sha` + # org.opencontainers.image.revision + # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys annotations: | ${{ steps.metadata.outputs.annotations }} - build.run-id=${{ github.run_id }} - build.run-attempt=${{ github.run_attempt }} - build.commit-sha=${{ steps.commit-sha.outputs.head }} + gha.run-id=${{ github.run_id }} + gha.run-attempt=${{ github.run_attempt }} # Required to be true so we can consistently get access to the `digest`: # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 push: true From fda1a0e62cf9141d3d81bf04ef3b6db60d846878 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:53:38 -0600 Subject: [PATCH 13/73] Debug --- action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index 0be0d3c..2ae1174 100644 --- a/action.yaml +++ b/action.yaml @@ -30,7 +30,8 @@ outputs: runs: using: composite steps: - - run: jq <<<"${{ toJSON(github.event) }}" + - shell: bash + run: jq <"${{ github.event.path }}" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: From 930cb44d3df71cf1fd7fa92b8156d100e4025cec Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 09:57:02 -0600 Subject: [PATCH 14/73] Debug --- action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index 2ae1174..0a0ebdb 100644 --- a/action.yaml +++ b/action.yaml @@ -31,7 +31,7 @@ runs: using: composite steps: - shell: bash - run: jq <"${{ github.event.path }}" + run: jq <"$GITHUB_EVENT_PATH" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: From 883d987c6fc414518730aa374daff5d37c8f7342 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:02:38 -0600 Subject: [PATCH 15/73] Iterating --- action.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index 0a0ebdb..fd2ad26 100644 --- a/action.yaml +++ b/action.yaml @@ -31,7 +31,12 @@ runs: using: composite steps: - shell: bash - run: jq <"$GITHUB_EVENT_PATH" + run: | + echo "${{ github.event.pull_request.head.sha }}" + echo "${{ github.event.pull_request.merge_commit_sha }}" + echo "${{ github.sha }}" + echo "$(git rev-parse HEAD)" + jq <"$GITHUB_EVENT_PATH" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: From 433a3a11e01c4c313b4e226e8bbed5edf1842075 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:05:15 -0600 Subject: [PATCH 16/73] Iterating --- action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/action.yaml b/action.yaml index fd2ad26..540e88e 100644 --- a/action.yaml +++ b/action.yaml @@ -69,7 +69,7 @@ runs: type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} env: # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: "true" + # DOCKER_METADATA_PR_HEAD_SHA: "true" # Use separate cache images to avoid bloating final images # https://docs.docker.com/build/cache/backends/registry/ - name: Docker cache-from @@ -85,7 +85,7 @@ runs: type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} env: # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: "true" + # DOCKER_METADATA_PR_HEAD_SHA: "true" - name: Docker cache-to id: cache-to uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 @@ -97,7 +97,7 @@ runs: type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} env: # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: "true" + # DOCKER_METADATA_PR_HEAD_SHA: "true" - name: Docker cache metadata id: cache shell: bash From ca0f5d96b220ae36f472e43b4588b157f128ba0e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:06:53 -0600 Subject: [PATCH 17/73] Iterating --- action.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/action.yaml b/action.yaml index 540e88e..c3960f6 100644 --- a/action.yaml +++ b/action.yaml @@ -67,9 +67,9 @@ runs: type=sha,prefix=sha- type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - env: - # https://github.com/docker/metadata-action/issues/206 - # DOCKER_METADATA_PR_HEAD_SHA: "true" + # env: + # # https://github.com/docker/metadata-action/issues/206 + # DOCKER_METADATA_PR_HEAD_SHA: "true" # Use separate cache images to avoid bloating final images # https://docs.docker.com/build/cache/backends/registry/ - name: Docker cache-from @@ -83,9 +83,9 @@ runs: type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} - env: - # https://github.com/docker/metadata-action/issues/206 - # DOCKER_METADATA_PR_HEAD_SHA: "true" + # env: + # # https://github.com/docker/metadata-action/issues/206 + # DOCKER_METADATA_PR_HEAD_SHA: "true" - name: Docker cache-to id: cache-to uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 @@ -95,9 +95,9 @@ runs: tags: | type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - env: - # https://github.com/docker/metadata-action/issues/206 - # DOCKER_METADATA_PR_HEAD_SHA: "true" + # env: + # # https://github.com/docker/metadata-action/issues/206 + # DOCKER_METADATA_PR_HEAD_SHA: "true" - name: Docker cache metadata id: cache shell: bash From c5df118a496b374a387ddb128e4a0fb3820b268d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:21:35 -0600 Subject: [PATCH 18/73] Fix SHA determination --- action.yaml | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/action.yaml b/action.yaml index c3960f6..b610f0f 100644 --- a/action.yaml +++ b/action.yaml @@ -48,6 +48,20 @@ runs: # Determine commit SHA head="$(git rev-parse HEAD)" echo "head=$head" | tee -a "$GITHUB_OUTPUT" + + case "$head" in + "${{ github.event.pull_request.head.sha }}") + is_pr_head_sha=true + ;; + "${{ github.sha }}") + is_pr_head_sha=false + ;; + *) + echo "Context uses unexpected commit SHA" >&2 + exit 1 + ;; + esac + echo "is-pr-head-sha=${is_pr_head_sha}" | tee -a "$GITHUB_OUTPUT" # Optional branch name (e.g. "main") for workflows triggered by `pull_request` or `push` events. - name: Branch id: branch @@ -67,9 +81,9 @@ runs: type=sha,prefix=sha- type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - # env: - # # https://github.com/docker/metadata-action/issues/206 - # DOCKER_METADATA_PR_HEAD_SHA: "true" + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} # Use separate cache images to avoid bloating final images # https://docs.docker.com/build/cache/backends/registry/ - name: Docker cache-from @@ -83,9 +97,9 @@ runs: type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} - # env: - # # https://github.com/docker/metadata-action/issues/206 - # DOCKER_METADATA_PR_HEAD_SHA: "true" + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} - name: Docker cache-to id: cache-to uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 @@ -95,9 +109,9 @@ runs: tags: | type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - # env: - # # https://github.com/docker/metadata-action/issues/206 - # DOCKER_METADATA_PR_HEAD_SHA: "true" + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} - name: Docker cache metadata id: cache shell: bash From 607b10c4f91a21a24b57c61152cdbb3755baf83a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:28:45 -0600 Subject: [PATCH 19/73] Test annotation --- .github/workflows/integration-tests.yaml | 38 ++++++++++++++++++++++-- action.yaml | 7 ----- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 438717d..1bba302 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -7,8 +7,8 @@ on: - ".github/workflows/integration-tests.yaml" jobs: - test: - name: Test + test-merge-commit: + name: Test Merge Commit # These permissions are needed to: # - Checkout the repo permissions: @@ -30,3 +30,37 @@ jobs: context: test build-args: | DEBIAN_VERSION=bookworm + - run: | + set -x + json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" + [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ github.sha }}" ]] || exit 1 + + test-head-commit: + name: Test Head Commit + # These permissions are needed to: + # - Checkout the repo + permissions: + contents: read + packages: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io/beacon-biosignals/whalesay + username: ${{ github.actor }} + password: ${{ github.token }} + - uses: ./ + id: build + with: + image-repository: ghcr.io/beacon-biosignals/whalesay + context: test + build-args: | + DEBIAN_VERSION=bookworm + - run: | + set -x + json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" + [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ github.pull_request.head.sha }}" ]] || exit 1 \ No newline at end of file diff --git a/action.yaml b/action.yaml index b610f0f..c3d12af 100644 --- a/action.yaml +++ b/action.yaml @@ -30,13 +30,6 @@ outputs: runs: using: composite steps: - - shell: bash - run: | - echo "${{ github.event.pull_request.head.sha }}" - echo "${{ github.event.pull_request.merge_commit_sha }}" - echo "${{ github.sha }}" - echo "$(git rev-parse HEAD)" - jq <"$GITHUB_EVENT_PATH" - name: Set up Docker Buildx uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: From 557b442cf7c1468d6bb1b2826c602973dcdbefc4 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:30:03 -0600 Subject: [PATCH 20/73] fixup! Test annotation --- .github/workflows/integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 1bba302..1491f02 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -63,4 +63,4 @@ jobs: - run: | set -x json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" - [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ github.pull_request.head.sha }}" ]] || exit 1 \ No newline at end of file + [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ github.event.pull_request.head.sha }}" ]] || exit 1 From e89070e7f86c5aaefcdfa516b8f828f0a438234e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:49:33 -0600 Subject: [PATCH 21/73] Expand tests --- .github/workflows/integration-tests.yaml | 55 +++++++++--------------- README.md | 9 +++- 2 files changed, 29 insertions(+), 35 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 1491f02..2949c3d 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -7,14 +7,22 @@ on: - ".github/workflows/integration-tests.yaml" jobs: - test-merge-commit: - name: Test Merge Commit + test: + name: Test ${{ matrix.commit.sha }} # These permissions are needed to: # - Checkout the repo permissions: contents: read packages: write runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + commit: + - title: Merge Commit + sha: ${{ github.sha }} + - title: Head Commit + sha: ${{ github.sha }} steps: - uses: actions/checkout@v4 - name: Log in to the Container registry @@ -30,37 +38,16 @@ jobs: context: test build-args: | DEBIAN_VERSION=bookworm - - run: | + - name: Validate image works + run: | + docker pull "${{ steps.build.outputs.image }}" + output="$(docker run "${{ steps.build.outputs.image }}")" + if [[ "$(wc -l <<<"$output")" -lt 15 ]]; then + echo "$output" + exit 1 + fi + - name: Validate annotations + run: | set -x json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" - [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ github.sha }}" ]] || exit 1 - - test-head-commit: - name: Test Head Commit - # These permissions are needed to: - # - Checkout the repo - permissions: - contents: read - packages: write - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io/beacon-biosignals/whalesay - username: ${{ github.actor }} - password: ${{ github.token }} - - uses: ./ - id: build - with: - image-repository: ghcr.io/beacon-biosignals/whalesay - context: test - build-args: | - DEBIAN_VERSION=bookworm - - run: | - set -x - json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" - [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ github.event.pull_request.head.sha }}" ]] || exit 1 + [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ matrix.commit.sha }}" ]] || exit 1 diff --git a/README.md b/README.md index a7087f8..105a295 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ We recommend utilizing a separate image repositories for deployment and producti --- jobs: example: + # These permissions are needed to: + # - Get the workflow run: https://github.com/beacon-biosignals/docker-build#permissions permissions: {} runs-on: ubuntu-latest steps: @@ -54,4 +56,9 @@ jobs: ## Permissions -No [job permissions](https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs) are required to run this action. +The follow [job permissions](https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs) are required to run this action: + +```yaml +permissions: + packages: write # Only required when using the GitHub Container registry: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry +``` \ No newline at end of file From 5790c41c68fd074ed9061eb7d542fbdf0070335b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 10:55:18 -0600 Subject: [PATCH 22/73] Validate build arg --- .github/workflows/integration-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 2949c3d..78ec649 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -37,7 +37,7 @@ jobs: image-repository: ghcr.io/beacon-biosignals/whalesay context: test build-args: | - DEBIAN_VERSION=bookworm + DEBIAN_VERSION=12.9 - name: Validate image works run: | docker pull "${{ steps.build.outputs.image }}" @@ -46,6 +46,8 @@ jobs: echo "$output" exit 1 fi + debian_version="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/debian_version)" + [[ "$debian_version" == "12.9" ]] || exit 2 - name: Validate annotations run: | set -x From bdb9cd5e992c789f54be22eddd4660bf5d310777 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:03:29 -0600 Subject: [PATCH 23/73] WIP --- .github/workflows/integration-tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 78ec649..0c6a993 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -48,6 +48,9 @@ jobs: fi debian_version="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/debian_version)" [[ "$debian_version" == "12.9" ]] || exit 2 + - name: Validate cache images + run: | + docker image inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" - name: Validate annotations run: | set -x From d692d6a71a4f3284946e6a24c2ac01768acd9569 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:04:59 -0600 Subject: [PATCH 24/73] Iterating --- .github/workflows/integration-tests.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 0c6a993..b949185 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -8,7 +8,7 @@ on: jobs: test: - name: Test ${{ matrix.commit.sha }} + name: Test ${{ matrix.commit.title }} # These permissions are needed to: # - Checkout the repo permissions: @@ -22,7 +22,7 @@ jobs: - title: Merge Commit sha: ${{ github.sha }} - title: Head Commit - sha: ${{ github.sha }} + sha: ${{ github.event.pull_request.head.sha }} steps: - uses: actions/checkout@v4 - name: Log in to the Container registry @@ -50,7 +50,7 @@ jobs: [[ "$debian_version" == "12.9" ]] || exit 2 - name: Validate cache images run: | - docker image inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" + docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" - name: Validate annotations run: | set -x From f97fce1f929e315483d0c17227d07d4b8759bfab Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:22:45 -0600 Subject: [PATCH 25/73] Iterating --- .github/workflows/integration-tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index b949185..7067a35 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -25,6 +25,8 @@ jobs: sha: ${{ github.event.pull_request.head.sha }} steps: - uses: actions/checkout@v4 + with: + ref: ${{ matrix.commit.sha }} - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -51,6 +53,7 @@ jobs: - name: Validate cache images run: | docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" + docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }}" - name: Validate annotations run: | set -x From eccc109aedb178d33f43b271f63127429e73d9c8 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:28:31 -0600 Subject: [PATCH 26/73] Iterating --- .github/workflows/integration-tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 7067a35..fb7994c 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -53,7 +53,9 @@ jobs: - name: Validate cache images run: | docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" - docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }}" + docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${branch//[^[:alnum:]]/_}" + env: + branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }}" - name: Validate annotations run: | set -x From ed849758e309569e37d11edec193cdb65a4bc69f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:41:19 -0600 Subject: [PATCH 27/73] Unset docker/metadata-action env variables --- action.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/action.yaml b/action.yaml index c3d12af..0d1fef1 100644 --- a/action.yaml +++ b/action.yaml @@ -105,6 +105,11 @@ runs: env: # https://github.com/docker/metadata-action/issues/206 DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} + # Remove environmental variables set by `docker/metadata-action`: + # https://github.com/docker/metadata-action?tab=readme-ov-file#outputs + - name: Unset metadata-action environment variables + shell: bash + run: sed -i "/^DOCKER_METADATA_OUTPUT_/d" "$GITHUB_ENV" - name: Docker cache metadata id: cache shell: bash From ecaee3f7609fd913d61cd62b0146e622bd99612d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:44:04 -0600 Subject: [PATCH 28/73] Debug --- action.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index 0d1fef1..3fe6a37 100644 --- a/action.yaml +++ b/action.yaml @@ -109,7 +109,10 @@ runs: # https://github.com/docker/metadata-action?tab=readme-ov-file#outputs - name: Unset metadata-action environment variables shell: bash - run: sed -i "/^DOCKER_METADATA_OUTPUT_/d" "$GITHUB_ENV" + run: | + echo "$GITHUB_ENV" + sed -i '/^DOCKER_METADATA_OUTPUT_/d' "$GITHUB_ENV" + cat "$GITHUB_ENV" - name: Docker cache metadata id: cache shell: bash From fd930bdce35807b5ec94ecee17f3c8e541626606 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:52:43 -0600 Subject: [PATCH 29/73] Overwrite docker/metadata-action env --- action.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/action.yaml b/action.yaml index 3fe6a37..5983454 100644 --- a/action.yaml +++ b/action.yaml @@ -105,14 +105,19 @@ runs: env: # https://github.com/docker/metadata-action/issues/206 DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} - # Remove environmental variables set by `docker/metadata-action`: - # https://github.com/docker/metadata-action?tab=readme-ov-file#outputs + # Disable environmental variables set by `docker/metadata-action`: + # https://github.com/docker/metadata-action#outputs + # https://github.com/docker/metadata-action/issues/490 - name: Unset metadata-action environment variables shell: bash run: | - echo "$GITHUB_ENV" - sed -i '/^DOCKER_METADATA_OUTPUT_/d' "$GITHUB_ENV" - cat "$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_VERSION=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_TAGS=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_ANNOTATIONS=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_JSON=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_TAGS=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_LABELS=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_ANNOTATIONS=" >>"$GITHUB_ENV" - name: Docker cache metadata id: cache shell: bash From 5d7dde08d2928c49ec5f30e3038a9c4e89053aa3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:55:30 -0600 Subject: [PATCH 30/73] fixup! Overwrite docker/metadata-action env --- action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/action.yaml b/action.yaml index 5983454..4593bff 100644 --- a/action.yaml +++ b/action.yaml @@ -113,11 +113,13 @@ runs: run: | echo "DOCKER_METADATA_OUTPUT_VERSION=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_TAGS=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_LABELS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_ANNOTATIONS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_JSON=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_TAGS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_LABELS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_ANNOTATIONS=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_BAKE_FILE=" >>"$GITHUB_ENV" - name: Docker cache metadata id: cache shell: bash From d20171e427432fa8ae4c923e754650ab0e674498 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:57:23 -0600 Subject: [PATCH 31/73] fixup --- .github/workflows/integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index fb7994c..75ef7af 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -55,7 +55,7 @@ jobs: docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${branch//[^[:alnum:]]/_}" env: - branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }}" + branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }} - name: Validate annotations run: | set -x From 3281e835bc72dc671db9dc1f9270e7b2d65c056e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 11:59:37 -0600 Subject: [PATCH 32/73] fixup --- .github/workflows/integration-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 75ef7af..e433bce 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -44,7 +44,7 @@ jobs: run: | docker pull "${{ steps.build.outputs.image }}" output="$(docker run "${{ steps.build.outputs.image }}")" - if [[ "$(wc -l <<<"$output")" -lt 15 ]]; then + if [[ "$(wc -l <<<"$output")" -lt 14 ]]; then echo "$output" exit 1 fi @@ -53,7 +53,7 @@ jobs: - name: Validate cache images run: | docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" - docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${branch//[^[:alnum:]]/_}" + docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${branch//[^[:alnum:]]/-}" env: branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }} - name: Validate annotations From a7ed0529d9a59889de5dd782aa33c7f6673b70b3 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 12:03:40 -0600 Subject: [PATCH 33/73] Use repository marked as temporary --- .github/workflows/integration-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index e433bce..ec4b4fd 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -30,13 +30,13 @@ jobs: - name: Log in to the Container registry uses: docker/login-action@v3 with: - registry: ghcr.io/beacon-biosignals/whalesay + registry: ghcr.io username: ${{ github.actor }} password: ${{ github.token }} - uses: ./ id: build with: - image-repository: ghcr.io/beacon-biosignals/whalesay + image-repository: ghcr.io/beacon-biosignals/temporary/whalesay context: test build-args: | DEBIAN_VERSION=12.9 From 6769abf8e1a34a8e714f1a0b09d1580047b6f0d8 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 13:17:57 -0600 Subject: [PATCH 34/73] Experiment with GHCR cleanup --- .github/workflows/integration-tests.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index ec4b4fd..85446d6 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -61,3 +61,14 @@ jobs: set -x json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ matrix.commit.sha }}" ]] || exit 1 + + cleanup: + name: Cleanup + needs: test + runs-on: ubuntu-latest + steps: + - uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b5dbf3a6f2cc869f0dbb0b8211f50c4 # v1.0.16 + with: + package: temporary/whalesay + older-than: 30 minutes + dry-run: true From 686c002b5bb7e5a7ed6b22ded3b35bb4b758a607 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 13:21:23 -0600 Subject: [PATCH 35/73] No longer a dry-run --- .github/workflows/integration-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 85446d6..1496043 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -71,4 +71,3 @@ jobs: with: package: temporary/whalesay older-than: 30 minutes - dry-run: true From 48fecc2ada4e8f7d903bc00816e16107cd737e1a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 13:25:36 -0600 Subject: [PATCH 36/73] fixup --- .github/workflows/integration-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 1496043..ffc503b 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -71,3 +71,5 @@ jobs: with: package: temporary/whalesay older-than: 30 minutes + delete-untagged: true + exclude-tags: branch-main,cache-branch-main From 25c053be8b33e326fd6a2af5fb3a5acfdfa5830a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 13:27:42 -0600 Subject: [PATCH 37/73] fixup --- .github/workflows/integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index ffc503b..d4dd81b 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -71,5 +71,5 @@ jobs: with: package: temporary/whalesay older-than: 30 minutes - delete-untagged: true + keep-n-tagged: 0 exclude-tags: branch-main,cache-branch-main From eee06c907572647c870977491e9e0bd87d7bc9a1 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 13:58:26 -0600 Subject: [PATCH 38/73] Test layer caching --- .github/workflows/integration-tests.yaml | 37 +++++++++++++++++++----- test/Dockerfile | 3 ++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index d4dd81b..b65f7e5 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -5,10 +5,14 @@ on: paths: - "action.yaml" - ".github/workflows/integration-tests.yaml" + push: + paths: + - "action.yaml" + - ".github/workflows/integration-tests.yaml" jobs: test: - name: Test ${{ matrix.commit.title }} + name: Test ${{ matrix.test.title }} # These permissions are needed to: # - Checkout the repo permissions: @@ -18,15 +22,21 @@ jobs: strategy: fail-fast: false matrix: - commit: + test: - title: Merge Commit - sha: ${{ github.sha }} + commit-sha: ${{ github.sha }} + from-scratch: true - title: Head Commit - sha: ${{ github.event.pull_request.head.sha }} + commit-sha: ${{ github.event.pull_request.head.sha }} steps: + - name: Job started at + id: job-started + run: | + job_started_at="$(date --utc --iso-8601=seconds)" + echo "at=$job_started_at" | tee -a "$GITHUB_OUTPUT" - uses: actions/checkout@v4 with: - ref: ${{ matrix.commit.sha }} + ref: ${{ matrix.test.commit-sha }} - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -40,6 +50,7 @@ jobs: context: test build-args: | DEBIAN_VERSION=12.9 + from-scratch: ${{ matrix.test.from-scratch || 'false' }} - name: Validate image works run: | docker pull "${{ steps.build.outputs.image }}" @@ -50,9 +61,19 @@ jobs: fi debian_version="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/debian_version)" [[ "$debian_version" == "12.9" ]] || exit 2 + - name: Validate layer caching + run: | + layer_created_at="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/layer-created-at)" + if [[ "$from_scratch" == "true" ]]; then + [[ "$(date -d "$layer_created_at" +%s)" -gt "$(date -d "$job_started_at" +%s)" ]] || exit 1 + else + [[ "$(date -d "$layer_created_at" +%s)" -lt "$(date -d "$job_started_at" +%s)" ]] || exit 1 + fi + env: + job_started_at: ${{ steps.job-started.outputs.at }} - name: Validate cache images run: | - docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.commit.sha }}" + docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.test.commit-sha }}" docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${branch//[^[:alnum:]]/-}" env: branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }} @@ -60,7 +81,7 @@ jobs: run: | set -x json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" - [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ matrix.commit.sha }}" ]] || exit 1 + [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ matrix.test.commit-sha }}" ]] || exit 1 cleanup: name: Cleanup @@ -70,6 +91,6 @@ jobs: - uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b5dbf3a6f2cc869f0dbb0b8211f50c4 # v1.0.16 with: package: temporary/whalesay - older-than: 30 minutes + older-than: 1 day keep-n-tagged: 0 exclude-tags: branch-main,cache-branch-main diff --git a/test/Dockerfile b/test/Dockerfile index 11a317d..7c26784 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -10,6 +10,9 @@ FROM debian:$DEBIAN_VERSION # Reduces output from `apt-get` ENV DEBIAN_FRONTEND="noninteractive" +# Used to validate Docker layer caching +RUN date --utc --iso-8601=seconds >/etc/layer-created-at + # Configure `apt-get` to keep downloaded packages. Needed for using `--mount=type=cache` with `apt-get` # https://docs.docker.com/engine/reference/python-deps/#example-cache-apt-packages RUN rm -f /etc/apt/apt.conf.d/docker-clean && \ From f15b05cca62f274ec0277b14f2ee618c71517968 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 14:01:41 -0600 Subject: [PATCH 39/73] fixup! Test layer caching --- .github/workflows/integration-tests.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index b65f7e5..6487370 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -6,6 +6,8 @@ on: - "action.yaml" - ".github/workflows/integration-tests.yaml" push: + branches: + - main paths: - "action.yaml" - ".github/workflows/integration-tests.yaml" @@ -63,6 +65,7 @@ jobs: [[ "$debian_version" == "12.9" ]] || exit 2 - name: Validate layer caching run: | + set -x layer_created_at="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/layer-created-at)" if [[ "$from_scratch" == "true" ]]; then [[ "$(date -d "$layer_created_at" +%s)" -gt "$(date -d "$job_started_at" +%s)" ]] || exit 1 From e788b95c30f448d6705dbc82eef1cdf5884627c6 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 14:06:10 -0600 Subject: [PATCH 40/73] Iterating --- .github/workflows/integration-tests.yaml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 6487370..26aec03 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -63,17 +63,25 @@ jobs: fi debian_version="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/debian_version)" [[ "$debian_version" == "12.9" ]] || exit 2 - - name: Validate layer caching + - name: Layer created at + id: layer-created run: | - set -x layer_created_at="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/layer-created-at)" - if [[ "$from_scratch" == "true" ]]; then - [[ "$(date -d "$layer_created_at" +%s)" -gt "$(date -d "$job_started_at" +%s)" ]] || exit 1 - else - [[ "$(date -d "$layer_created_at" +%s)" -lt "$(date -d "$job_started_at" +%s)" ]] || exit 1 - fi + echo "at=$layer_created_at" | tee -a "$GITHUB_OUTPUT" + - name: Validate layer caching + if: ${{ matrix.test.from-scratch == 'false' }} + run: | + [[ "$(date -d "$layer_created_at" +%s)" -lt "$(date -d "$job_started_at" +%s)" ]] || exit 1 + env: + job_started_at: ${{ steps.job-started.outputs.at }} + layer_created_at: ${{ steps.layer-created.outputs.at }} + - name: Validate no layer caching + if: ${{ matrix.test.from-scratch == 'true' }} + run: | + [[ "$(date -d "$layer_created_at" +%s)" -gt "$(date -d "$job_started_at" +%s)" ]] || exit 1 env: job_started_at: ${{ steps.job-started.outputs.at }} + layer_created_at: ${{ steps.layer-created.outputs.at } - name: Validate cache images run: | docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.test.commit-sha }}" From 6570cb8c2af3928774e07e9f97ebb814eba9bf7c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 14:06:54 -0600 Subject: [PATCH 41/73] fixup! Iterating --- .github/workflows/integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 26aec03..93e8e0c 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -81,7 +81,7 @@ jobs: [[ "$(date -d "$layer_created_at" +%s)" -gt "$(date -d "$job_started_at" +%s)" ]] || exit 1 env: job_started_at: ${{ steps.job-started.outputs.at }} - layer_created_at: ${{ steps.layer-created.outputs.at } + layer_created_at: ${{ steps.layer-created.outputs.at }} - name: Validate cache images run: | docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.test.commit-sha }}" From 0fc558ea9a6b3e3f67c7ee0db80066978a765ae5 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 14:08:55 -0600 Subject: [PATCH 42/73] fixup! Iterating --- .github/workflows/integration-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 93e8e0c..b271857 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -69,14 +69,14 @@ jobs: layer_created_at="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/layer-created-at)" echo "at=$layer_created_at" | tee -a "$GITHUB_OUTPUT" - name: Validate layer caching - if: ${{ matrix.test.from-scratch == 'false' }} + if: ${{ matrix.test.from-scratch == false }} run: | [[ "$(date -d "$layer_created_at" +%s)" -lt "$(date -d "$job_started_at" +%s)" ]] || exit 1 env: job_started_at: ${{ steps.job-started.outputs.at }} layer_created_at: ${{ steps.layer-created.outputs.at }} - name: Validate no layer caching - if: ${{ matrix.test.from-scratch == 'true' }} + if: ${{ matrix.test.from-scratch == true }} run: | [[ "$(date -d "$layer_created_at" +%s)" -gt "$(date -d "$job_started_at" +%s)" ]] || exit 1 env: From e556a7e481dd6a7473f7d4304826743cd544bec5 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 14:22:16 -0600 Subject: [PATCH 43/73] Additional details in README --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 105a295..1295744 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,21 @@ # Docker Build -Build a Docker image while utilizing [layer caching](https://docs.docker.com/build/cache/) -backed from the image repository. Although [Docker does support using GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) -as a layer cache backend but the GHA cache limit for a repository is 10 GB which is -quite limiting for Docker images. +Build a Docker image while utilizing [layer caching](https://docs.docker.com/build/cache/) backed from the image repository. Image tags will be automatically created based upon the relevant PR, branch name, and commit SHA. -We recommend utilizing a separate image repositories for deployment and production (e.g.`temporary/my-image` and `permanent/my-image`) to make it easier to separate temporary images from permanent images meant for end users. Promoting temporary images to be permanent can be done with `docker push` or [`regctl image copy --digest-tags`](https://github.com/regclient/regclient/blob/main/docs/regctl.md#registry-commands) if you want the digest to be identical across registries. +When using this action we recommend utilizing a separate image repositories for development and production (e.g.`temporary/my-image` and `permanent/my-image`) to make it easier to separate temporary images from permanent images meant for end users. The `beacon-biosignals/docker-build` action is used to build temporary images under development. Once a temporary image is ready for production it can be promoted to be permanent by using `docker tag`/`docker push` or [`regctl image copy --digest-tags`](https://github.com/regclient/regclient/blob/main/docs/regctl.md#registry-commands) (if you want the digest to be identical across registries) to transfer the image. + +Note that although [Docker does support using GitHub Actions cache](https://docs.docker.com/build/cache/backends/gha/) as a layer cache backend the GHA cache limit for a repository is 10 GB which is quite limiting for larger Docker images. ## Example ```yaml --- +on: + pull_request: {} + # Trigger this build workflow on "main". See `from-scratch` + push: + branches: + - main jobs: example: # These permissions are needed to: @@ -30,7 +35,7 @@ jobs: # Example of passing in Docker `--secret` build-secrets: | github-token=${{ secrets.token || github.token }} - # Build images from scratch on `main`. Ensures system packages have latest security fixes. + # Build images from scratch on "main". Ensures that caching doesn't result in using insecure system packages. from-scratch: ${{ github.ref == 'refs/heads/main' }} ``` From edabbb85aef1f9da5cfe45f2985e784ac0d00ad1 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Tue, 14 Jan 2025 14:26:10 -0600 Subject: [PATCH 44/73] More descriptions --- README.md | 2 +- action.yaml | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1295744..60589d0 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ jobs: | `image-repository` | The Docker image repository where the image was pushed to. | `temporary/my-image` | | `digest` | The built Docker image digest. | `sha256:37782d4e1c24d8f12047039a0d3512d1b6059e306a80d5b66a1d9ff60247a8cb` | | `tags` | JSON list of tags associated with the built Docker image. | `branch-main`, `sha-152cb14` | -| `commit-sha` | The Git commit SHA used to build the imag. | `152cb14643b50529b229930d6124e6bbef48668d` | +| `commit-sha` | The Git commit SHA used to build the image. | `152cb14643b50529b229930d6124e6bbef48668d` | ## Permissions diff --git a/action.yaml b/action.yaml index 4593bff..20f54b5 100644 --- a/action.yaml +++ b/action.yaml @@ -6,26 +6,35 @@ branding: icon: layers inputs: image-repository: + description: The Docker image repository to push the build image and cached layers. required: true context: + description: The Docker build context directory. default: "." build-args: + description: List of build-time variables. required: false build-secrets: + description: List of secrets to expose to the build. required: false from-scratch: - description: Do not read from the cache when building the image + description: Do not read from the cache when building the image. default: "false" outputs: image: + description: Reference to the build image including the digest. value: ${{ inputs.image-repository }}@${{ steps.build-push.outputs.digest }} image-repository: + description: The Docker image repository where the image was pushed to. value: ${{ inputs.image-repository }} digest: + description: The built Docker image digest. value: ${{ steps.build-push.outputs.digest }} tags: + description: JSON list of tags associated with the built Docker image. value: ${{ steps.tags.outputs.json }} commit-sha: + description: The Git commit SHA used to build the image. value: ${{ github.event.pull_request.head.sha || github.sha }} runs: using: composite From 951fe7ca626eb3316125f559af1799cf5fdf6768 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 10:03:39 -0600 Subject: [PATCH 45/73] Add comment about test limitations with a single image repo --- .github/workflows/integration-tests.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index b271857..6d455f1 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -25,6 +25,20 @@ jobs: fail-fast: false matrix: test: + # We need to avoid running concurrent tests using the same commit SHA as + # otherwise we could see tests pass when one of them doesn't say output + # the cache layers. We could address this in two ways in the future if this + # becomes limiting: + # + # 1. Make use of separate image repositories for each test. This allows each + # test to run in parallel without the potential for conflicts. + # 2. Use job concurrency and `max-parallel` for matrix jobs to run jobs + # sequentially with cleanup in between. May be rather slow. + # + # I also considered revising the action to avoid pushing images entirely. + # Doing this may be challenging in otherways as pushing the image is a + # requirement for getting the digests in some contexts: + # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 - title: Merge Commit commit-sha: ${{ github.sha }} from-scratch: true From b4c5d962e343eaab41c7b35fe3f007e67edc01ca Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 10:06:08 -0600 Subject: [PATCH 46/73] Try git context --- action.yaml | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/action.yaml b/action.yaml index 20f54b5..1394ebb 100644 --- a/action.yaml +++ b/action.yaml @@ -1,4 +1,4 @@ ---- +a--- name: Docker Build description: Build a Docker image while utilize layer caching from the image repository. branding: @@ -50,20 +50,8 @@ runs: # Determine commit SHA head="$(git rev-parse HEAD)" echo "head=$head" | tee -a "$GITHUB_OUTPUT" - - case "$head" in - "${{ github.event.pull_request.head.sha }}") - is_pr_head_sha=true - ;; - "${{ github.sha }}") - is_pr_head_sha=false - ;; - *) - echo "Context uses unexpected commit SHA" >&2 - exit 1 - ;; - esac - echo "is-pr-head-sha=${is_pr_head_sha}" | tee -a "$GITHUB_OUTPUT" + echo "pr.head.sha=${{ github.event.pull_request.head.sha }}" + echo "github.sha=${{ github.sha }}" # Optional branch name (e.g. "main") for workflows triggered by `pull_request` or `push` events. - name: Branch id: branch @@ -77,15 +65,13 @@ runs: id: metadata uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: + context: git images: | ${{ inputs.image-repository }} tags: | type=sha,prefix=sha- type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - env: - # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} # Use separate cache images to avoid bloating final images # https://docs.docker.com/build/cache/backends/registry/ - name: Docker cache-from @@ -93,27 +79,23 @@ runs: if: ${{ inputs.from-scratch != 'true' }} uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: + context: git images: | ${{ inputs.image-repository }} tags: | type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} - env: - # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} - name: Docker cache-to id: cache-to uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: + context: git images: | ${{ inputs.image-repository }} tags: | type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - env: - # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} # Disable environmental variables set by `docker/metadata-action`: # https://github.com/docker/metadata-action#outputs # https://github.com/docker/metadata-action/issues/490 From 5921a42f27af154dec1372cb1e1d1fe11c701437 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 10:07:39 -0600 Subject: [PATCH 47/73] fixup! Try git context --- action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yaml b/action.yaml index 1394ebb..ca6f208 100644 --- a/action.yaml +++ b/action.yaml @@ -1,4 +1,4 @@ -a--- +--- name: Docker Build description: Build a Docker image while utilize layer caching from the image repository. branding: From ec612d574fffff205b5f74d06a58b935a5f14e53 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 10:53:45 -0600 Subject: [PATCH 48/73] Revert "Try git context" This reverts commit b4c5d962e343eaab41c7b35fe3f007e67edc01ca. --- action.yaml | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/action.yaml b/action.yaml index ca6f208..20f54b5 100644 --- a/action.yaml +++ b/action.yaml @@ -50,8 +50,20 @@ runs: # Determine commit SHA head="$(git rev-parse HEAD)" echo "head=$head" | tee -a "$GITHUB_OUTPUT" - echo "pr.head.sha=${{ github.event.pull_request.head.sha }}" - echo "github.sha=${{ github.sha }}" + + case "$head" in + "${{ github.event.pull_request.head.sha }}") + is_pr_head_sha=true + ;; + "${{ github.sha }}") + is_pr_head_sha=false + ;; + *) + echo "Context uses unexpected commit SHA" >&2 + exit 1 + ;; + esac + echo "is-pr-head-sha=${is_pr_head_sha}" | tee -a "$GITHUB_OUTPUT" # Optional branch name (e.g. "main") for workflows triggered by `pull_request` or `push` events. - name: Branch id: branch @@ -65,13 +77,15 @@ runs: id: metadata uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: - context: git images: | ${{ inputs.image-repository }} tags: | type=sha,prefix=sha- type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} # Use separate cache images to avoid bloating final images # https://docs.docker.com/build/cache/backends/registry/ - name: Docker cache-from @@ -79,23 +93,27 @@ runs: if: ${{ inputs.from-scratch != 'true' }} uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: - context: git images: | ${{ inputs.image-repository }} tags: | type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} - name: Docker cache-to id: cache-to uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 with: - context: git images: | ${{ inputs.image-repository }} tags: | type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} # Disable environmental variables set by `docker/metadata-action`: # https://github.com/docker/metadata-action#outputs # https://github.com/docker/metadata-action/issues/490 From a22e8126fbb891f0187072fe1ab2e95c9db8fdad Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 10:54:43 -0600 Subject: [PATCH 49/73] Try an arbitrary SHA --- action.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/action.yaml b/action.yaml index 20f54b5..be67355 100644 --- a/action.yaml +++ b/action.yaml @@ -55,12 +55,8 @@ runs: "${{ github.event.pull_request.head.sha }}") is_pr_head_sha=true ;; - "${{ github.sha }}") - is_pr_head_sha=false - ;; *) - echo "Context uses unexpected commit SHA" >&2 - exit 1 + is_pr_head_sha=false ;; esac echo "is-pr-head-sha=${is_pr_head_sha}" | tee -a "$GITHUB_OUTPUT" From 4078dbf050123eb10f8713f477918a0e5f61f2da Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 10:55:23 -0600 Subject: [PATCH 50/73] fixup! Try an arbitrary SHA --- .github/workflows/integration-tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 6d455f1..6a9334c 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -44,6 +44,8 @@ jobs: from-scratch: true - title: Head Commit commit-sha: ${{ github.event.pull_request.head.sha }} + - title: Fixed Commit + commit-sha: 5921a42f27af154dec1372cb1e1d1fe11c701437 steps: - name: Job started at id: job-started From 592535024cec02ff60dbd021c8a44b99b3fd7260 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 11:04:29 -0600 Subject: [PATCH 51/73] Support arbitrary commits --- action.yaml | 39 +++++++++++++-------------------------- 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/action.yaml b/action.yaml index be67355..6b0ff2f 100644 --- a/action.yaml +++ b/action.yaml @@ -35,7 +35,7 @@ outputs: value: ${{ steps.tags.outputs.json }} commit-sha: description: The Git commit SHA used to build the image. - value: ${{ github.event.pull_request.head.sha || github.sha }} + value: ${{ steps.commit-head.outputs.sha }} runs: using: composite steps: @@ -43,23 +43,19 @@ runs: uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: driver: docker-container + # Avoiding using `docker/metadata-action`'s `type=sha` as it is limited to either + # `github.sha` or `github.event.pull_request.head.sha` (when `DOCKER_METADATA_PR_HEAD_SHA=true`). + # https://github.com/docker/metadata-action/issues/206 + # https://github.com/docker/metadata-action/issues/362 - name: Determine commit SHA - id: commit-sha + id: commit-head shell: bash run: | # Determine commit SHA - head="$(git rev-parse HEAD)" - echo "head=$head" | tee -a "$GITHUB_OUTPUT" - - case "$head" in - "${{ github.event.pull_request.head.sha }}") - is_pr_head_sha=true - ;; - *) - is_pr_head_sha=false - ;; - esac - echo "is-pr-head-sha=${is_pr_head_sha}" | tee -a "$GITHUB_OUTPUT" + sha="$(git rev-parse HEAD)" + short_sha="$(git rev-parse --short "$sha")" + echo "sha=$sha" | tee -a "$GITHUB_OUTPUT" + echo "short-sha=$short_sha" | tee -a "$GITHUB_OUTPUT" # Optional branch name (e.g. "main") for workflows triggered by `pull_request` or `push` events. - name: Branch id: branch @@ -76,12 +72,9 @@ runs: images: | ${{ inputs.image-repository }} tags: | - type=sha,prefix=sha- + type=raw,prefix=sha-,value=${{ steps.commit-head.outputs.short-sha }} type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - env: - # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} # Use separate cache images to avoid bloating final images # https://docs.docker.com/build/cache/backends/registry/ - name: Docker cache-from @@ -92,12 +85,9 @@ runs: images: | ${{ inputs.image-repository }} tags: | - type=sha,prefix=cache-sha-,format=long + type=raw,prefix=cache-sha-,value=${{ steps.commit-head.outputs.sha }} type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} - env: - # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} - name: Docker cache-to id: cache-to uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 @@ -105,11 +95,8 @@ runs: images: | ${{ inputs.image-repository }} tags: | - type=sha,prefix=cache-sha-,format=long + type=raw,prefix=cache-sha-,value=${{ steps.commit-head.outputs.sha }} type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} - env: - # https://github.com/docker/metadata-action/issues/206 - DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit-sha.outputs.is-pr-head-sha }} # Disable environmental variables set by `docker/metadata-action`: # https://github.com/docker/metadata-action#outputs # https://github.com/docker/metadata-action/issues/490 From 98b26c0e0950ee41fa7483e32c9031148abf2dd8 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 11:13:54 -0600 Subject: [PATCH 52/73] Cleanup overrides --- .github/workflows/integration-tests.yaml | 7 +++++++ action.yaml | 7 +++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 6a9334c..1760d10 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -109,6 +109,13 @@ jobs: set -x json="$(docker manifest inspect "${{ steps.build.outputs.image }}")" [[ "$(jq -r '.annotations."org.opencontainers.image.revision"' <<<"$json")" == "${{ matrix.test.commit-sha }}" ]] || exit 1 + - name: Validate docker/metadata-output environment variables are overwritten + shell: bash + run: | + if [[ "$(printenv | grep '^DOCKER_METADATA_OUTPUT_' | grep '[^=]$' | wc -l)" -ne 0 ]]; then + printenv | grep '^DOCKER_METADATA_OUTPUT_' + exit 1 + fi cleanup: name: Cleanup diff --git a/action.yaml b/action.yaml index 6b0ff2f..cae875b 100644 --- a/action.yaml +++ b/action.yaml @@ -67,7 +67,7 @@ runs: branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }} - name: Docker metadata id: metadata - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 with: images: | ${{ inputs.image-repository }} @@ -80,7 +80,7 @@ runs: - name: Docker cache-from id: cache-from if: ${{ inputs.from-scratch != 'true' }} - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 with: images: | ${{ inputs.image-repository }} @@ -90,7 +90,7 @@ runs: type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} - name: Docker cache-to id: cache-to - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 with: images: | ${{ inputs.image-repository }} @@ -111,7 +111,6 @@ runs: echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_TAGS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_LABELS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_ANNOTATIONS=" >>"$GITHUB_ENV" - echo "DOCKER_METADATA_OUTPUT_BAKE_FILE=" >>"$GITHUB_ENV" - name: Docker cache metadata id: cache shell: bash From efd8837f31ce4c6410c82c2fa4645a9b79d28021 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 11:17:45 -0600 Subject: [PATCH 53/73] Back to restricted SHA --- .github/workflows/integration-tests.yaml | 2 -- action.yaml | 39 +++++++++++++++++------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 1760d10..9712710 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -44,8 +44,6 @@ jobs: from-scratch: true - title: Head Commit commit-sha: ${{ github.event.pull_request.head.sha }} - - title: Fixed Commit - commit-sha: 5921a42f27af154dec1372cb1e1d1fe11c701437 steps: - name: Job started at id: job-started diff --git a/action.yaml b/action.yaml index cae875b..163bad7 100644 --- a/action.yaml +++ b/action.yaml @@ -35,7 +35,7 @@ outputs: value: ${{ steps.tags.outputs.json }} commit-sha: description: The Git commit SHA used to build the image. - value: ${{ steps.commit-head.outputs.sha }} + value: ${{ steps.commit.outputs.sha }} runs: using: composite steps: @@ -43,19 +43,27 @@ runs: uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0 with: driver: docker-container - # Avoiding using `docker/metadata-action`'s `type=sha` as it is limited to either - # `github.sha` or `github.event.pull_request.head.sha` (when `DOCKER_METADATA_PR_HEAD_SHA=true`). - # https://github.com/docker/metadata-action/issues/206 - # https://github.com/docker/metadata-action/issues/362 - name: Determine commit SHA - id: commit-head + id: commit shell: bash run: | # Determine commit SHA sha="$(git rev-parse HEAD)" - short_sha="$(git rev-parse --short "$sha")" echo "sha=$sha" | tee -a "$GITHUB_OUTPUT" - echo "short-sha=$short_sha" | tee -a "$GITHUB_OUTPUT" + + case "$sha" in + "${{ github.event.pull_request.head.sha }}") + is_pr_head_sha=true + ;; + "${{ github.sha }}") + is_pr_head_sha=false + ;; + *) + echo "Context uses unexpected commit SHA" >&2 + exit 1 + ;; + esac + echo "is-pr-head-sha=${is_pr_head_sha}" | tee -a "$GITHUB_OUTPUT" # Optional branch name (e.g. "main") for workflows triggered by `pull_request` or `push` events. - name: Branch id: branch @@ -72,9 +80,12 @@ runs: images: | ${{ inputs.image-repository }} tags: | - type=raw,prefix=sha-,value=${{ steps.commit-head.outputs.short-sha }} + type=sha,prefix=sha-,format=short type=ref,prefix=pr-,event=pr type=raw,prefix=branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit.outputs.is-pr-head-sha }} # Use separate cache images to avoid bloating final images # https://docs.docker.com/build/cache/backends/registry/ - name: Docker cache-from @@ -85,9 +96,12 @@ runs: images: | ${{ inputs.image-repository }} tags: | - type=raw,prefix=cache-sha-,value=${{ steps.commit-head.outputs.sha }} + type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} type=raw,prefix=cache-sha-,value=${{ github.event.pull_request.base.sha }},enable=${{ github.event_name == 'pull_request' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit.outputs.is-pr-head-sha }} - name: Docker cache-to id: cache-to uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96 # v5.6.1 @@ -95,8 +109,11 @@ runs: images: | ${{ inputs.image-repository }} tags: | - type=raw,prefix=cache-sha-,value=${{ steps.commit-head.outputs.sha }} + type=sha,prefix=cache-sha-,format=long type=raw,prefix=cache-branch-,value=${{ steps.branch.outputs.name }},enable=${{ steps.branch.outputs.name != '' }} + env: + # https://github.com/docker/metadata-action/issues/206 + DOCKER_METADATA_PR_HEAD_SHA: ${{ steps.commit.outputs.is-pr-head-sha }} # Disable environmental variables set by `docker/metadata-action`: # https://github.com/docker/metadata-action#outputs # https://github.com/docker/metadata-action/issues/490 From 074aa3db9ae337442b7bbb83279199f152340e5f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 11:27:30 -0600 Subject: [PATCH 54/73] fixup! Cleanup overrides --- .github/workflows/integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 9712710..1ed135d 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -110,7 +110,7 @@ jobs: - name: Validate docker/metadata-output environment variables are overwritten shell: bash run: | - if [[ "$(printenv | grep '^DOCKER_METADATA_OUTPUT_' | grep '[^=]$' | wc -l)" -ne 0 ]]; then + if [[ "$(printenv | grep '^DOCKER_METADATA_OUTPUT_' | grep -c '[^=]$')" -ne 0 ]]; then printenv | grep '^DOCKER_METADATA_OUTPUT_' exit 1 fi From cb809bee0504e8d5bdefc185053b2320ff9adcc1 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 11:28:38 -0600 Subject: [PATCH 55/73] fixup! Cleanup overrides --- action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/action.yaml b/action.yaml index 163bad7..5659520 100644 --- a/action.yaml +++ b/action.yaml @@ -128,6 +128,7 @@ runs: echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_TAGS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_LABELS=" >>"$GITHUB_ENV" echo "DOCKER_METADATA_OUTPUT_BAKE_FILE_ANNOTATIONS=" >>"$GITHUB_ENV" + echo "DOCKER_METADATA_OUTPUT_BAKE_FILE=" >>"$GITHUB_ENV" - name: Docker cache metadata id: cache shell: bash From 18bcc6f5a497104590f22a82c9ff79dcba70d2c0 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 14:14:35 -0600 Subject: [PATCH 56/73] Add LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..868a0f7 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Beacon Biosignals + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 3a7af71cb6b36ce67e7beaebe38c96e6a2c6d604 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Wed, 15 Jan 2025 14:15:58 -0600 Subject: [PATCH 57/73] Update LICENSE year --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 868a0f7..df26dd2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2024 Beacon Biosignals +Copyright (c) 2025 Beacon Biosignals Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 409084e66fed42840be8b210a725ea2e1f24213c Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Thu, 16 Jan 2025 16:05:42 -0600 Subject: [PATCH 58/73] Comment on limited commit SHAs --- action.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/action.yaml b/action.yaml index 5659520..d9ca98b 100644 --- a/action.yaml +++ b/action.yaml @@ -51,6 +51,13 @@ runs: sha="$(git rev-parse HEAD)" echo "sha=$sha" | tee -a "$GITHUB_OUTPUT" + # When building the Docker image we'll generate tags and annotations which include + # the commit SHA. Due to limitations with `docker/metadata-action` it's rather + # difficult to get this working for an arbitrary commit. For now we'll limit this + # action to only either using the PR merge commit (default when checking out) or + # the PR/branch/tag head. Updating this action to support arbitrary actions can + # be made into a feature release. + # https://github.com/beacon-biosignals/docker-build/issues/2 case "$sha" in "${{ github.event.pull_request.head.sha }}") is_pr_head_sha=true From 6634dad78fe2e65d6f2514428cfec7f99a07b69f Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 17 Jan 2025 09:13:29 -0600 Subject: [PATCH 59/73] drop! Test push trigger --- .github/workflows/integration-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 1ed135d..d342c9e 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -6,8 +6,8 @@ on: - "action.yaml" - ".github/workflows/integration-tests.yaml" push: - branches: - - main + # branches: + # - main paths: - "action.yaml" - ".github/workflows/integration-tests.yaml" From 8c0ecfd6a3c22d320387b2e5ef8f2c6eec29489b Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 17 Jan 2025 09:20:43 -0600 Subject: [PATCH 60/73] Revise tests for push --- .github/workflows/integration-tests.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index d342c9e..6dfe811 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -39,11 +39,15 @@ jobs: # Doing this may be challenging in otherways as pushing the image is a # requirement for getting the digests in some contexts: # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 - - title: Merge Commit + - title: ${{ github.event_name == 'pull_request' && 'PR Merge Commit' || 'Head Commit' }} commit-sha: ${{ github.sha }} from-scratch: true - - title: Head Commit + - title: PR Head Commit commit-sha: ${{ github.event.pull_request.head.sha }} + exclude: + # Drop this test for non-PR events where the expression is empty + - title: PR Head Commit + commit-sha: "" steps: - name: Job started at id: job-started From 183adcf8c065da89ac6ed7c9f240ae01ff67f6d7 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 17 Jan 2025 09:20:59 -0600 Subject: [PATCH 61/73] drop! PR event for now --- .github/workflows/integration-tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 6dfe811..791db5f 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -1,10 +1,10 @@ --- name: Integration Tests on: - pull_request: - paths: - - "action.yaml" - - ".github/workflows/integration-tests.yaml" + # pull_request: + # paths: + # - "action.yaml" + # - ".github/workflows/integration-tests.yaml" push: # branches: # - main From 53ca0e40319823f85ce737e690b169af20fe476e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Fri, 17 Jan 2025 09:27:12 -0600 Subject: [PATCH 62/73] Test push in PRs --- .github/workflows/integration-tests.yaml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 791db5f..a768605 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -1,13 +1,11 @@ --- name: Integration Tests on: - # pull_request: - # paths: - # - "action.yaml" - # - ".github/workflows/integration-tests.yaml" + pull_request: + paths: + - "action.yaml" + - ".github/workflows/integration-tests.yaml" push: - # branches: - # - main paths: - "action.yaml" - ".github/workflows/integration-tests.yaml" @@ -39,15 +37,9 @@ jobs: # Doing this may be challenging in otherways as pushing the image is a # requirement for getting the digests in some contexts: # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 - - title: ${{ github.event_name == 'pull_request' && 'PR Merge Commit' || 'Head Commit' }} + - title: ${{ github.event_name == 'pull_request' && 'Merge Commit' || 'Head Commit' }} commit-sha: ${{ github.sha }} - from-scratch: true - - title: PR Head Commit - commit-sha: ${{ github.event.pull_request.head.sha }} - exclude: - # Drop this test for non-PR events where the expression is empty - - title: PR Head Commit - commit-sha: "" + from-scratch: ${{ github.event_name != 'pull_request' }} steps: - name: Job started at id: job-started From 3cf9d0c880a326f733309742b78027b1553ef9e5 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 14:27:37 -0600 Subject: [PATCH 63/73] Improve test parallelism --- .github/workflows/integration-tests.yaml | 97 +++++++++++++++++++----- 1 file changed, 76 insertions(+), 21 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index a768605..54acc3f 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -10,9 +10,70 @@ on: - "action.yaml" - ".github/workflows/integration-tests.yaml" +concurrency: + group: integration-tests-${{ github.sha }} + cancel-in-progress: true + jobs: + filter-matrix: + name: Filter Matrix + runs-on: ubuntu-latest + outputs: + json: ${{ steps.filter.outputs.json }} + steps: + - run: | + # Remove any entries with keys containing `null` values. + output_yaml="$(yq 'map(select(to_entries | map(.value != null) | all))' <<<"${matrix:?}")" + + # Output our multiline YAML document using GH action flavored heredoc + # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings + { + echo "json< Date: Mon, 20 Jan 2025 14:29:55 -0600 Subject: [PATCH 64/73] fixup! Improve test parallelism --- .github/workflows/integration-tests.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 54acc3f..ff8b9f8 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -21,7 +21,9 @@ jobs: outputs: json: ${{ steps.filter.outputs.json }} steps: - - run: | + - name: Filter Matrix + id: filter + run: | # Remove any entries with keys containing `null` values. output_yaml="$(yq 'map(select(to_entries | map(.value != null) | all))' <<<"${matrix:?}")" @@ -65,11 +67,11 @@ jobs: - title: ${{ github.event_name == 'pull_request' && 'Merge Commit From Scratch' || '' }} package: temporary/whalesay-from-scratch commit-sha: ${{ github.sha }} - from-scratch: false + from-scratch: true - title: Head Commit From Scratch package: temporary/whalesay-from-scratch commit-sha: ${{ github.event.pull_request.head.sha || github.sha }} - from-scratch: false + from-scratch: true test: name: Test ${{ matrix.test.title }} From ead4c73369fd8a3cd03d90be805040a5ec54d971 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 14:31:19 -0600 Subject: [PATCH 65/73] fixup! Improve test parallelism --- .github/workflows/integration-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index ff8b9f8..ff51c56 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -85,7 +85,7 @@ jobs: strategy: fail-fast: false matrix: - test: ${{ needs.filter-matrix.outputs.json }} + test: ${{ fromJSON(needs.filter-matrix.outputs.json) }} steps: - name: Job started at id: job-started @@ -171,7 +171,7 @@ jobs: strategy: fail-fast: false matrix: - package: ${{ needs.filter-matrix.outputs.packages }} + package: ${{ fromJSON(needs.filter-matrix.outputs.packages) }} steps: - uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b5dbf3a6f2cc869f0dbb0b8211f50c4 # v1.0.16 with: From 04ffd2fbceb424e6eb2466e08bc7df9ef2da6974 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 14:53:39 -0600 Subject: [PATCH 66/73] Remove cache layers during cleanup --- .github/workflows/integration-tests.yaml | 31 +++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index ff51c56..e1d6593 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -19,23 +19,26 @@ jobs: name: Filter Matrix runs-on: ubuntu-latest outputs: - json: ${{ steps.filter.outputs.json }} + test-json: ${{ steps.filter.outputs.test-json }} + cleanup-json: ${{ steps.filter.outputs.cleanup-json }} steps: - name: Filter Matrix id: filter run: | # Remove any entries with keys containing `null` values. - output_yaml="$(yq 'map(select(to_entries | map(.value != null) | all))' <<<"${matrix:?}")" + test_yaml="$(yq 'map(select(to_entries | map(.value != null) | all))' <<<"${matrix:?}")" + + cleanup_yaml="$(yq 'group_by(.package) | map({"package": .[0].package, "tags" : map(.commit-sha) | unique | map("cache-sha-" + .) | join(",")})' <<<"$test_yaml")" # Output our multiline YAML document using GH action flavored heredoc # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings { - echo "json< Date: Mon, 20 Jan 2025 15:00:51 -0600 Subject: [PATCH 67/73] Separate PR/push repos --- .github/workflows/integration-tests.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index e1d6593..f6c8f35 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -60,19 +60,19 @@ jobs: # https://github.com/docker/build-push-action/issues/906#issuecomment-1674567311 matrix: | - title: ${{ github.event_name == 'pull_request' && 'Merge Commit' || '' }} - package : temporary/whalesay + package : temporary/whalesay-pr commit-sha: ${{ github.sha }} from-scratch: false - title: Head Commit - package: temporary/whalesay + package: temporary/whalesay-${{ github.event_name == 'pull_request' && 'pr' || 'push' }} commit-sha: ${{ github.event.pull_request.head.sha || github.sha }} from-scratch: false - title: ${{ github.event_name == 'pull_request' && 'Merge Commit From Scratch' || '' }} - package: temporary/whalesay-from-scratch + package: temporary/whalesay-pr-from-scratch commit-sha: ${{ github.sha }} from-scratch: true - title: Head Commit From Scratch - package: temporary/whalesay-from-scratch + package: temporary/whalesay-${{ github.event_name == 'pull_request' && 'pr' || 'push' }}-from-scratch commit-sha: ${{ github.event.pull_request.head.sha || github.sha }} from-scratch: true From c78889ca6810dfcfce26cd9359bf8f2f81499afd Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 15:04:58 -0600 Subject: [PATCH 68/73] Empty: test From cc6e18d0fdd8ae2df0d40debb5500994c904dd0e Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 15:10:57 -0600 Subject: [PATCH 69/73] Update concurrency group --- .github/workflows/integration-tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index f6c8f35..fc21fae 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -11,7 +11,7 @@ on: - ".github/workflows/integration-tests.yaml" concurrency: - group: integration-tests-${{ github.sha }} + group: integration-tests-${{ github.event_name == 'pull_request' && 'pr' || 'push' }}-${{ github.event.pull_request.head.sha || github.sha }} cancel-in-progress: true jobs: @@ -127,6 +127,7 @@ jobs: run: | layer_created_at="$(docker run --entrypoint=/bin/cat "${{ steps.build.outputs.image }}" /etc/layer-created-at)" echo "at=$layer_created_at" | tee -a "$GITHUB_OUTPUT" + # Test will fail if this is the first time the image was build in the image-repository - name: Validate layer caching if: ${{ matrix.test.from-scratch == false }} run: | From 57e377b63a0d27861761ca10aa28a5cda6cad09a Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 15:23:37 -0600 Subject: [PATCH 70/73] Add safety check --- .github/workflows/integration-tests.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index fc21fae..bfae355 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -24,10 +24,15 @@ jobs: steps: - name: Filter Matrix id: filter + shell: bash run: | # Remove any entries with keys containing `null` values. test_yaml="$(yq 'map(select(to_entries | map(.value != null) | all))' <<<"${matrix:?}")" + # Validate we do not accidentally test against the same package and commit SHA. + yq -o=json <<<"$output_yaml" | jq -e '(map({package, "commit-sha"}) | unique | length) == length' || exit 1 + + # Automatically cleanup the `cache-sha-*` tags for the specific test commits. cleanup_yaml="$(yq 'group_by(.package) | map({"package": .[0].package, "tags" : map(.commit-sha) | unique | map("cache-sha-" + .) | join(",")})' <<<"$test_yaml")" # Output our multiline YAML document using GH action flavored heredoc From 78dd073780199abaa933e90489ffee57e6f22f14 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 15:27:56 -0600 Subject: [PATCH 71/73] fixup --- .github/workflows/integration-tests.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index bfae355..1815cf9 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -30,7 +30,7 @@ jobs: test_yaml="$(yq 'map(select(to_entries | map(.value != null) | all))' <<<"${matrix:?}")" # Validate we do not accidentally test against the same package and commit SHA. - yq -o=json <<<"$output_yaml" | jq -e '(map({package, "commit-sha"}) | unique | length) == length' || exit 1 + yq -o=json <<<"$test_yaml" | jq -e '(map({package, "commit-sha"}) | unique | length) == length' || exit 1 # Automatically cleanup the `cache-sha-*` tags for the specific test commits. cleanup_yaml="$(yq 'group_by(.package) | map({"package": .[0].package, "tags" : map(.commit-sha) | unique | map("cache-sha-" + .) | join(",")})' <<<"$test_yaml")" @@ -56,8 +56,10 @@ jobs: # or different Git commit SHAs. # 2. Utilizing concurrency groups to avoid having multiple instances of this # workflow run in parallel when triggered on the same commit SHA. - # 3. Deleting the `cache-sha-*` entry for the images we're about to build to - # ensure those entries were produced by the running workflow. + # 3. Deleting the `cache-sha-*` tags to ensure our running workflow produced + # those images. Ideally, we'd delete these before the tests run but attempting + # to delete images from non-existing packages causes failures so this works + # well enough. # # I also considered revising the action to avoid pushing images entirely. # Doing this may be challenging in otherways as pushing the image is a From 18b6aaabac2afa194be2709badccbc92437fe3c8 Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 15:29:58 -0600 Subject: [PATCH 72/73] Simplify cleanup name --- .github/workflows/integration-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 1815cf9..e0ae16e 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -169,7 +169,7 @@ jobs: fi cleanup: - name: Cleanup + name: Cleanup (${{ matrix.cleanup.package }}) needs: - filter-matrix - test From 703c6b54da62d45821326361cd2f71becac2aa0d Mon Sep 17 00:00:00 2001 From: Curtis Vogt Date: Mon, 20 Jan 2025 15:37:07 -0600 Subject: [PATCH 73/73] Tag fix --- .github/workflows/integration-tests.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index e0ae16e..222fb01 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -152,7 +152,11 @@ jobs: - name: Validate cache images run: | docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-sha-${{ matrix.test.commit-sha }}" - docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${branch//[^[:alnum:]]/-}" + + # Should only be skipped when workflow is triggered by a tag push + if [[ -n "$branch" ]]; then + docker manifest inspect "${{ steps.build.outputs.image-repository }}:cache-branch-${branch//[^[:alnum:]]/-}" + fi env: branch: ${{ github.head_ref || (github.ref_type == 'branch' && github.ref_name) }} - name: Validate annotations