From 68f4ffad6f3d13ff14dd738a2174dbc2e1ab79ca Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Sun, 10 Sep 2023 21:09:37 +0200 Subject: [PATCH] CI: Add GHA workflows to build and publish OCI container images to GHCR --- .github/workflows/oci-examples.yml | 145 +++++++++++++++++++++++++++++ .github/workflows/oci-server.yml | 145 +++++++++++++++++++++++++++++ release/oci-examples/Dockerfile | 43 +++++++++ release/oci-examples/selftest.sh | 11 +++ release/oci-examples/test.yml | 4 + release/oci-server/Dockerfile | 39 ++++++++ release/oci-server/selftest.sh | 11 +++ release/oci-server/test.yml | 4 + 8 files changed, 402 insertions(+) create mode 100644 .github/workflows/oci-examples.yml create mode 100644 .github/workflows/oci-server.yml create mode 100644 release/oci-examples/Dockerfile create mode 100755 release/oci-examples/selftest.sh create mode 100644 release/oci-examples/test.yml create mode 100644 release/oci-server/Dockerfile create mode 100755 release/oci-server/selftest.sh create mode 100644 release/oci-server/test.yml diff --git a/.github/workflows/oci-examples.yml b/.github/workflows/oci-examples.yml new file mode 100644 index 0000000..daeef0e --- /dev/null +++ b/.github/workflows/oci-examples.yml @@ -0,0 +1,145 @@ +# Stage OCI container images through GitHub Actions (GHA) to GitHub Container Registry (GHCR). +name: OCI for examples + +on: + pull_request: ~ + push: + tags: + - '*.*.*' + + # Produce a nightly image every day at 6 a.m. CEST. + schedule: + - cron: '0 4 * * *' + + # Allow job to be triggered manually. + workflow_dispatch: + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +# The name for the produced image at ghcr.io. +env: + IMAGE_NAME: "${{ github.repository }}-examples" + RECIPE_PATH: "release/oci-examples" + +jobs: + build_and_test: + runs-on: ubuntu-latest + + steps: + - name: Acquire sources + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: "x64" + cache: "pip" + cache-dependency-path: "pyproject.toml" + + - name: Build wheel package + run: | + pip install build + python -m build + + - name: Upload wheel package + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-wheel-${{ github.sha }} + path: dist/*.whl + retention-days: 7 + + - name: Run tests + run: | + if [[ -f ${{ env.RECIPE_PATH }}/test.yml ]]; then + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + docker-compose --file ${{ env.RECIPE_PATH }}/test.yml build + docker-compose --file ${{ env.RECIPE_PATH }}/test.yml run sut + fi + + oci: + needs: build_and_test + runs-on: ubuntu-latest + if: ${{ ! (startsWith(github.actor, 'dependabot') || github.event.pull_request.head.repo.fork ) }} + + steps: + - name: Acquire sources + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Define image name and tags + id: meta + uses: docker/metadata-action@v4 + with: + # List of OCI images to use as base name for tags + images: | + ghcr.io/${{ env.IMAGE_NAME }} + # Generate OCI image tags based on the following events/attributes + tags: | + type=schedule,pattern=nightly + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Inspect metadata + run: | + echo "Tags: ${{ steps.meta.outputs.tags }}" + echo "Labels: ${{ steps.meta.outputs.labels }}" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache OCI layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Inspect builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Build and push image + uses: docker/build-push-action@v4 + with: + context: . + file: ${{ env.RECIPE_PATH }}/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Display git status + run: | + set -x + git describe --tags --always + git status diff --git a/.github/workflows/oci-server.yml b/.github/workflows/oci-server.yml new file mode 100644 index 0000000..c91d4d5 --- /dev/null +++ b/.github/workflows/oci-server.yml @@ -0,0 +1,145 @@ +# Stage OCI container images through GitHub Actions (GHA) to GitHub Container Registry (GHCR). +name: OCI for server + +on: + pull_request: ~ + push: + tags: + - '*.*.*' + + # Produce a nightly image every day at 6 a.m. CEST. + schedule: + - cron: '0 4 * * *' + + # Allow job to be triggered manually. + workflow_dispatch: + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +# The name for the produced image at ghcr.io. +env: + IMAGE_NAME: "${{ github.repository }}" + RECIPE_PATH: "release/oci-server" + +jobs: + build_and_test: + runs-on: ubuntu-latest + + steps: + - name: Acquire sources + uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: "x64" + cache: "pip" + cache-dependency-path: "pyproject.toml" + + - name: Build wheel package + run: | + pip install build + python -m build + + - name: Upload wheel package + uses: actions/upload-artifact@v3 + with: + name: ${{ runner.os }}-wheel-${{ github.sha }} + path: dist/*.whl + retention-days: 7 + + - name: Run tests + run: | + if [[ -f ${{ env.RECIPE_PATH }}/test.yml ]]; then + export DOCKER_BUILDKIT=1 + export COMPOSE_DOCKER_CLI_BUILD=1 + docker-compose --file ${{ env.RECIPE_PATH }}/test.yml build + docker-compose --file ${{ env.RECIPE_PATH }}/test.yml run sut + fi + + oci: + needs: build_and_test + runs-on: ubuntu-latest + if: ${{ ! (startsWith(github.actor, 'dependabot') || github.event.pull_request.head.repo.fork ) }} + + steps: + - name: Acquire sources + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Define image name and tags + id: meta + uses: docker/metadata-action@v4 + with: + # List of OCI images to use as base name for tags + images: | + ghcr.io/${{ env.IMAGE_NAME }} + # Generate OCI image tags based on the following events/attributes + tags: | + type=schedule,pattern=nightly + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Inspect metadata + run: | + echo "Tags: ${{ steps.meta.outputs.tags }}" + echo "Labels: ${{ steps.meta.outputs.labels }}" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Cache OCI layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: Inspect builder + run: | + echo "Name: ${{ steps.buildx.outputs.name }}" + echo "Endpoint: ${{ steps.buildx.outputs.endpoint }}" + echo "Status: ${{ steps.buildx.outputs.status }}" + echo "Flags: ${{ steps.buildx.outputs.flags }}" + echo "Platforms: ${{ steps.buildx.outputs.platforms }}" + + - name: Login to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Build and push image + uses: docker/build-push-action@v4 + with: + context: . + file: ${{ env.RECIPE_PATH }}/Dockerfile + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + push: true + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Display git status + run: | + set -x + git describe --tags --always + git status diff --git a/release/oci-examples/Dockerfile b/release/oci-examples/Dockerfile new file mode 100644 index 0000000..9878537 --- /dev/null +++ b/release/oci-examples/Dockerfile @@ -0,0 +1,43 @@ +# syntax=docker/dockerfile:1 + +# Use BuildKit's build-time cache mounts, it makes a huge difference on rebuilds. +# - https://vsupalov.com/buildkit-cache-mount-dockerfile/ +# - https://github.com/FernandoMiguel/Buildkit#mounttypecache + +FROM python:3.11-slim-bullseye + +ENV DEBIAN_FRONTEND noninteractive +ENV TERM linux + +# Install Git, it is needed for `versioningit`. +RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + +# Install distribution packages, with caching. +RUN --mount=type=cache,id=apt,target=/var/cache/apt --mount=type=cache,id=apt,target=/var/lib/apt \ + true \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests --yes git + +# Copy sources +COPY . /src + +# Install package, with caching of dependency packages. +RUN --mount=type=cache,id=pip,target=/root/.cache/pip \ + pip install --use-pep517 --prefer-binary '/src[examples]' + +# Uninstall Git again. +RUN apt-get --yes remove --purge git && apt-get --yes autoremove + +# Purge /src and /tmp directories. +RUN rm -rf /src /tmp/* + +# Copy selftest.sh to the image +COPY release/oci-examples/selftest.sh /usr/local/bin + +# Run program once, in order to let Python compile its files. +RUN mlflow-cratedb --version +RUN mlflow-cratedb cratedb --version + +# Copy example programs and documentation to the image. +COPY examples /opt/ml-examples/bin +COPY docs /opt/ml-examples/docs diff --git a/release/oci-examples/selftest.sh b/release/oci-examples/selftest.sh new file mode 100755 index 0000000..9bad244 --- /dev/null +++ b/release/oci-examples/selftest.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Fail on error. +set -e + +# Display all commands. +# set -x + +echo "Invoking MLflow adapter for CrateDB" +mflow-cratedb --version +mflow-cratedb cratedb --version diff --git a/release/oci-examples/test.yml b/release/oci-examples/test.yml new file mode 100644 index 0000000..2bdf7bd --- /dev/null +++ b/release/oci-examples/test.yml @@ -0,0 +1,4 @@ +sut: + build: ../.. + dockerfile: release/oci-examples/Dockerfile + command: selftest.sh diff --git a/release/oci-server/Dockerfile b/release/oci-server/Dockerfile new file mode 100644 index 0000000..bf0a6d4 --- /dev/null +++ b/release/oci-server/Dockerfile @@ -0,0 +1,39 @@ +# syntax=docker/dockerfile:1 + +# Use BuildKit's build-time cache mounts, it makes a huge difference on rebuilds. +# - https://vsupalov.com/buildkit-cache-mount-dockerfile/ +# - https://github.com/FernandoMiguel/Buildkit#mounttypecache + +FROM python:3.11-slim-bullseye + +ENV DEBIAN_FRONTEND noninteractive +ENV TERM linux + +# Install Git, it is needed for `versioningit`. +RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache + +# Install distribution packages, with caching. +RUN --mount=type=cache,id=apt,target=/var/cache/apt --mount=type=cache,id=apt,target=/var/lib/apt \ + true \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests --yes git + +# Copy sources +COPY . /src + +# Install package, with caching of dependency packages. +RUN --mount=type=cache,id=pip,target=/root/.cache/pip \ + pip install --use-pep517 --prefer-binary '/src' + +# Uninstall Git again. +RUN apt-get --yes remove --purge git && apt-get --yes autoremove + +# Purge /src and /tmp directories. +RUN rm -rf /src /tmp/* + +# Copy selftest.sh to the image +COPY release/oci-server/selftest.sh /usr/local/bin + +# Run program once, in order to let Python compile its files. +RUN mlflow-cratedb --version +RUN mlflow-cratedb cratedb --version diff --git a/release/oci-server/selftest.sh b/release/oci-server/selftest.sh new file mode 100755 index 0000000..9bad244 --- /dev/null +++ b/release/oci-server/selftest.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Fail on error. +set -e + +# Display all commands. +# set -x + +echo "Invoking MLflow adapter for CrateDB" +mflow-cratedb --version +mflow-cratedb cratedb --version diff --git a/release/oci-server/test.yml b/release/oci-server/test.yml new file mode 100644 index 0000000..ee53bf9 --- /dev/null +++ b/release/oci-server/test.yml @@ -0,0 +1,4 @@ +sut: + build: ../.. + dockerfile: release/oci-server/Dockerfile + command: selftest.sh