From 3a631a1e5c73ab7dfeb1674b3f142570449aa884 Mon Sep 17 00:00:00 2001 From: bloodearnest Date: Tue, 4 Jun 2024 14:13:42 +0100 Subject: [PATCH 1/9] Add 24.04 base/actions image --- .github/workflows/build_and_publish.yaml | 4 ++++ Makefile | 2 +- docker-compose.yaml | 15 ++++++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_publish.yaml b/.github/workflows/build_and_publish.yaml index 2a61a5b..6ec72dc 100644 --- a/.github/workflows/build_and_publish.yaml +++ b/.github/workflows/build_and_publish.yaml @@ -32,14 +32,18 @@ jobs: # tag the local images with the published names docker tag $BASE_IMAGE_NAME:20.04 $BASE_IMAGE_ID:20.04 docker tag $BASE_IMAGE_NAME:22.04 $BASE_IMAGE_ID:22.04 + docker tag $BASE_IMAGE_NAME:24.04 $BASE_IMAGE_ID:24.04 docker tag $ACTION_IMAGE_NAME:20.04 $ACTION_IMAGE_ID:20.04 docker tag $ACTION_IMAGE_NAME:22.04 $ACTION_IMAGE_ID:22.04 + docker tag $ACTION_IMAGE_NAME:24.04 $ACTION_IMAGE_ID:24.04 # push each label up docker push $BASE_IMAGE_ID:20.04 docker push $BASE_IMAGE_ID:22.04 + docker push $BASE_IMAGE_ID:24.04 docker push $ACTION_IMAGE_ID:20.04 docker push $ACTION_IMAGE_ID:22.04 + docker push $ACTION_IMAGE_ID:24.04 # latest tags are 20.04 for b/w compat docker tag $BASE_IMAGE_NAME:20.04 $BASE_IMAGE_ID:latest diff --git a/Makefile b/Makefile index 7edc6af..488ec09 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ export BASE_GITREF=$(shell git rev-parse --short HEAD) build: - docker compose build --pull $(ARGS) base-docker-20.04 base-docker-22.04 base-action-20.04 base-action-22.04 + docker compose build --pull $(ARGS) clean-build: ARGS=--no-cache clean-build: build diff --git a/docker-compose.yaml b/docker-compose.yaml index be25db4..f0588b3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,6 +27,13 @@ services: args: - UBUNTU_VERSION=ubuntu:22.04 + base-docker-24.04: + extends: base-docker + image: "base-docker:24.04" + build: + args: + - UBUNTU_VERSION=ubuntu:24.04 + base-action-20.04: extends: base-docker image: "base-action:20.04" @@ -43,4 +50,10 @@ services: - UBUNTU_VERSION=ubuntu:22.04 target: base-action - + base-action-24.04: + extends: base-docker + image: "base-action:24.04" + build: + args: + - UBUNTU_VERSION=ubuntu:24.04 + target: base-action From 6e9d8ec5c0597d26157dda15ba76d9911ed4913f Mon Sep 17 00:00:00 2001 From: bloodearnest Date: Tue, 4 Jun 2024 14:08:29 +0100 Subject: [PATCH 2/9] Update apt caching setup based on latest docker docs https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages We automatically disable apt cleanup for all child images, on the assumption they will be using cache mounts. --- Dockerfile | 5 +++-- docker-apt-install.sh | 5 +---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1063a7e..a770e26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,8 +16,7 @@ LABEL org.opencontainers.image.authors="tech@opensafely.org" \ # Disable automatic cache cleaning, and make `apt install` preserve caches. # This implies we should always use RUN --mount=cache on apt installs -# Taken from: -# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#example-cache-apt-packages +# Taken from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache # useful utility for installing apt packages in the most space efficient way @@ -27,7 +26,9 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa COPY docker-apt-install.sh /root/docker-apt-install.sh # install some base tools we want in all images +# caching from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ UPGRADE=yes /root/docker-apt-install.sh ca-certificates sysstat lsof net-tools tcpdump vim strace file # record build info so downstream images know about the base image they were diff --git a/docker-apt-install.sh b/docker-apt-install.sh index 36aaa07..653e21f 100755 --- a/docker-apt-install.sh +++ b/docker-apt-install.sh @@ -21,11 +21,8 @@ for arg in "$@"; do fi done +# shellcheck disable=SC2086 test -n "$PACKAGES" && apt-get install --yes --no-install-recommends $PACKAGES # clean up if we've upgraded test "${UPGRADE:-}" = "yes" && apt-get autoremove --yes - -# We do not apt-get clean becuase the default debian docker apt config does that for us. -# Doing this saves us ~50MB, but means we need to apt-get update before we can install anything again -rm -rf /var/lib/apt/lists/* From caaff50ea777b2f8dcc4fd2a20a0cb63cb56045e Mon Sep 17 00:00:00 2001 From: bloodearnest Date: Tue, 22 Oct 2024 14:55:44 +0100 Subject: [PATCH 3/9] Fix lables to not have - character Annoyingly, the `docker inspect --format` syntax doesn't support - characters in labels. So rename `build-date` to `created`, and `vcs-ref` to gitref --- Dockerfile | 4 ++-- check.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index a770e26..28e2f2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,8 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ # built from ARG BASE_BUILD_DATE ARG BASE_GITREF -LABEL org.opensafely.base.build-date=$BASE_BUILD_DATE \ - org.opensafely.base.vcs-ref=$BASE_GITREF +LABEL org.opensafely.base.created=$BASE_BUILD_DATE \ + org.opensafely.base.gitref=$BASE_GITREF FROM base-docker as base-action diff --git a/check.sh b/check.sh index 26d82ae..04919df 100755 --- a/check.sh +++ b/check.sh @@ -9,7 +9,7 @@ do echo "$image: OS $os_version" test "$os_version" = "VERSION_ID=\"${tag}\"" || { failed=1; echo "Expected os version to be $tag"; } - for label in build-date vcs-ref + for label in created gitref do full="org.opensafely.base.$label" value="$(docker inspect -f "{{ index .Config.Labels \"$full\" }}" "$image")" From 16d5920554f9e9d03ae9ac5042df841634770e09 Mon Sep 17 00:00:00 2001 From: bloodearnest Date: Tue, 22 Oct 2024 16:16:22 +0100 Subject: [PATCH 4/9] Update to dockerfile 1.10 and add build checks There were no outstanding lint issues, but will help catch in future. --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 28e2f2c..3c4b822 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ -# syntax=docker/dockerfile:1.2 +# syntax=docker/dockerfile:1.10 +# enable docker linting +# check=error=true ARG UBUNTU_VERSION=ubuntu:20.04 # we are parameterizing the base image, so we can't be explicit like DL3006 wants us to be # hadolint ignore=DL3006 From 62e3cb850cf1aa2eafa22f7c9dd5eacd3e7f00e7 Mon Sep 17 00:00:00 2001 From: bloodearnest Date: Tue, 22 Oct 2024 14:54:44 +0100 Subject: [PATCH 5/9] Reduce the number of layers by one by combining RUN lines Uses the new HEREDOC syntax supported by the more recent dockerfile version. --- Dockerfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3c4b822..34b22a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,11 +16,6 @@ LABEL org.opencontainers.image.authors="tech@opensafely.org" \ org.opencontainers.image.vendor="OpenSAFELY" \ org.opencontainers.image.source="https://github.com/opensafely-core/base-docker" -# Disable automatic cache cleaning, and make `apt install` preserve caches. -# This implies we should always use RUN --mount=cache on apt installs -# Taken from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages -RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache - # useful utility for installing apt packages in the most space efficient way # possible. It's worth it because this is the base image, and so any bloat # here affects all our images. Plus, it's then available for downstream images @@ -28,10 +23,14 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloa COPY docker-apt-install.sh /root/docker-apt-install.sh # install some base tools we want in all images -# caching from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - UPGRADE=yes /root/docker-apt-install.sh ca-certificates sysstat lsof net-tools tcpdump vim strace file +# Ccaching from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages +# Enable full caching of apt packages and metadata, undoing the debian defaults. +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked < /etc/apt/apt.conf.d/keep-cache + UPGRADE=yes /root/docker-apt-install.sh ca-certificates sysstat lsof net-tools tcpdump vim strace file +EOF + # record build info so downstream images know about the base image they were # built from From 6411326cfb9aed20865c0d433e0a9d5fd40c7fdc Mon Sep 17 00:00:00 2001 From: bloodearnest Date: Wed, 23 Oct 2024 11:49:18 +0100 Subject: [PATCH 6/9] Extend tests/check.sh to cover new 24.04 images --- Justfile | 1 + check.sh | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/Justfile b/Justfile index a03a4d5..270387f 100644 --- a/Justfile +++ b/Justfile @@ -23,6 +23,7 @@ test: build fi docker run $RUN_ARGS --rm -v {{justfile_directory()}}:/tests -w /tests $ACTION_IMAGE_NAME:20.04 ./tests.sh docker run $RUN_ARGS --rm -v {{justfile_directory()}}:/tests -w /tests $ACTION_IMAGE_NAME:22.04 ./tests.sh + docker run $RUN_ARGS --rm -v {{justfile_directory()}}:/tests -w /tests $ACTION_IMAGE_NAME:24.04 ./tests.sh ./check.sh # Update the files tracking the SHAs of ubuntu docker image diff --git a/check.sh b/check.sh index 04919df..f23cfa6 100755 --- a/check.sh +++ b/check.sh @@ -2,20 +2,24 @@ set -euo pipefail failed=0 -for image in base-docker:20.04 base-docker:22.04 base-action:20.04 -do - tag=$(echo $image | awk -F: '{print $NF}') - os_version=$(docker run $image grep VERSION_ID= /etc/os-release) - echo "$image: OS $os_version" - test "$os_version" = "VERSION_ID=\"${tag}\"" || { failed=1; echo "Expected os version to be $tag"; } - - for label in created gitref +for version in 20.04 22.04 24.04 +do + for image_name in base-docker base-action do - full="org.opensafely.base.$label" - value="$(docker inspect -f "{{ index .Config.Labels \"$full\" }}" "$image")" + image="$image_name:$version" + tag=$(echo $image | awk -F: '{print $NF}') + os_version=$(docker run $image grep VERSION_ID= /etc/os-release) + echo "$image: OS $os_version" + test "$os_version" = "VERSION_ID=\"${tag}\"" || { failed=1; echo "Expected os version to be $tag"; } + + for label in created gitref + do + full="org.opensafely.base.$label" + value="$(docker inspect -f "{{ index .Config.Labels \"$full\" }}" "$image")" - echo "$image: $full=$value" - test -n "$value" || { failed=1; echo "Empty $full label"; } + echo "$image: $full=$value" + test -n "$value" || { failed=1; echo "Empty $full label"; } + done done done From 214e92732cd95dfa4841041d06e8aecd6752a76e Mon Sep 17 00:00:00 2001 From: bloodearnest Date: Wed, 23 Oct 2024 12:17:15 +0100 Subject: [PATCH 7/9] Fix and rename build date - rename from BASE_BUILD_DATE to BASE_CREATED to match label name - noticed it wasn't a valid utc iso timestamp, so fixed that - noticed we are using a weird hybrid of make and just, so fixed that --- .github/workflows/tests.yaml | 2 +- Dockerfile | 4 ++-- Justfile | 12 ++++++++++-- Makefile | 12 ------------ docker-compose.yaml | 2 +- 5 files changed, 14 insertions(+), 18 deletions(-) delete mode 100644 Makefile diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b76093d..611cf0a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -10,7 +10,7 @@ jobs: - name: Checkout uses: actions/checkout@master - name: Build image - run: make build + run: just build - uses: extractions/setup-just@69d82fb0233557aec017ef13706851d0694e0f1d - name: Run tests run: just test diff --git a/Dockerfile b/Dockerfile index 34b22a6..d6e6f7d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,9 +34,9 @@ EOF # record build info so downstream images know about the base image they were # built from -ARG BASE_BUILD_DATE +ARG BASE_CREATED ARG BASE_GITREF -LABEL org.opensafely.base.created=$BASE_BUILD_DATE \ +LABEL org.opensafely.base.created=$BASE_CREATED \ org.opensafely.base.gitref=$BASE_GITREF FROM base-docker as base-action diff --git a/Justfile b/Justfile index 270387f..824426a 100644 --- a/Justfile +++ b/Justfile @@ -3,14 +3,22 @@ export ACTION_IMAGE_NAME := env_var_or_default('ACTION_IMAGE_NAME', "base-action _default: @just --list -build: - make build +# build all images +build *args: + #!/bin/bash + export DOCKER_BUILDKIT=1 + export BASE_CREATED=$(date --utc +'%Y-%m-%dT%H:%M:%S+00:00') + export BASE_GITREF=$(git rev-parse --short HEAD) + docker compose build --pull {{ args }} + +clean-build: (build "--no-cache") # hadolint the Dockerfile lint: @docker pull hadolint/hadolint @docker run --rm -i hadolint/hadolint < Dockerfile +# build and test all images test: build #!/bin/bash set -euxo pipefail diff --git a/Makefile b/Makefile deleted file mode 100644 index 488ec09..0000000 --- a/Makefile +++ /dev/null @@ -1,12 +0,0 @@ -BASE_IMAGE_NAME ?= base-docker - -export DOCKER_BUILDKIT=1 -export BASE_BUILD_DATE=$(shell date +'%y-%m-%dT%H:%M:%S.%3NZ') -export BASE_GITREF=$(shell git rev-parse --short HEAD) - - -build: - docker compose build --pull $(ARGS) - -clean-build: ARGS=--no-cache -clean-build: build diff --git a/docker-compose.yaml b/docker-compose.yaml index f0588b3..9d9228f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,7 +9,7 @@ services: # this makes the image work for later cache_from: usage - BUILDKIT_INLINE_CACHE=1 # env vars supplied by make/just - - BASE_BUILD_DATE + - BASE_CREATED - BASE_GITREF init: true From dc4da3744cb1a01cae7b0c11855cec1f3b2ddb20 Mon Sep 17 00:00:00 2001 From: Simon Davy Date: Wed, 23 Oct 2024 12:23:48 +0100 Subject: [PATCH 8/9] typo fix Co-authored-by: Tom Ward --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d6e6f7d..a83ea50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ LABEL org.opencontainers.image.authors="tech@opensafely.org" \ COPY docker-apt-install.sh /root/docker-apt-install.sh # install some base tools we want in all images -# Ccaching from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages +# Caching from docs: https://docs.docker.com/reference/dockerfile/#example-cache-apt-packages # Enable full caching of apt packages and metadata, undoing the debian defaults. RUN --mount=type=cache,target=/var/cache/apt,sharing=locked --mount=type=cache,target=/var/lib/apt,sharing=locked < Date: Wed, 23 Oct 2024 12:17:15 +0100 Subject: [PATCH 9/9] Fix and rename build date - rename from BASE_BUILD_DATE to BASE_CREATED to match label name - noticed it wasn't a valid utc iso timestamp, so fixed that - noticed we are using a weird hybrid of make and just, so fixed that --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 611cf0a..65f71ab 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -9,9 +9,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@master + - uses: extractions/setup-just@69d82fb0233557aec017ef13706851d0694e0f1d - name: Build image run: just build - - uses: extractions/setup-just@69d82fb0233557aec017ef13706851d0694e0f1d - name: Run tests run: just test - name: Run lint