CI: Add GHA workflows to build and publish OCI container images to GHCR

crate · Sep 10, 2023 · 68f4ffa · 68f4ffa
1 parent 26e9b0b
commit 68f4ffa
Show file tree

Hide file tree

Showing 8 changed files with 402 additions and 0 deletions.
diff --git a/.github/workflows/oci-examples.yml b/.github/workflows/oci-examples.yml
@@ -0,0 +1,145 @@
+# Stage OCI container images through GitHub Actions (GHA) to GitHub Container Registry (GHCR).
+name: OCI for examples
+
+on:
+  pull_request: ~
+  push:
+    tags:
+      - '*.*.*'
+
+  # Produce a nightly image every day at 6 a.m. CEST.
+  schedule:
+    - cron: '0 4 * * *'
+
+  # Allow job to be triggered manually.
+  workflow_dispatch:
+
+# Cancel in-progress jobs when pushing to the same branch.
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+# The name for the produced image at ghcr.io.
+env:
+  IMAGE_NAME: "${{ github.repository }}-examples"
+  RECIPE_PATH: "release/oci-examples"
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Acquire sources
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+          cache: "pip"
+          cache-dependency-path: "pyproject.toml"
+
+      - name: Build wheel package
+        run: |
+          pip install build
+          python -m build
+
+      - name: Upload wheel package
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ runner.os }}-wheel-${{ github.sha }}
+          path: dist/*.whl
+          retention-days: 7
+
+      - name: Run tests
+        run: |
+          if [[ -f ${{ env.RECIPE_PATH }}/test.yml ]]; then
+            export DOCKER_BUILDKIT=1
+            export COMPOSE_DOCKER_CLI_BUILD=1
+            docker-compose --file ${{ env.RECIPE_PATH }}/test.yml build
+            docker-compose --file ${{ env.RECIPE_PATH }}/test.yml run sut
+          fi
+
+  oci:
+    needs: build_and_test
+    runs-on: ubuntu-latest
+    if: ${{ ! (startsWith(github.actor, 'dependabot') || github.event.pull_request.head.repo.fork ) }}
+
+    steps:
+      - name: Acquire sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Define image name and tags
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          # List of OCI images to use as base name for tags
+          images: |
+            ghcr.io/${{ env.IMAGE_NAME }}
+          # Generate OCI image tags based on the following events/attributes
+          tags: |
+            type=schedule,pattern=nightly
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Inspect metadata
+        run: |
+          echo "Tags:      ${{ steps.meta.outputs.tags }}"
+          echo "Labels:    ${{ steps.meta.outputs.labels }}"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Cache OCI layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.RECIPE_PATH }}/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Display git status
+        run: |
+          set -x
+          git describe --tags --always
+          git status
diff --git a/.github/workflows/oci-server.yml b/.github/workflows/oci-server.yml
@@ -0,0 +1,145 @@
+# Stage OCI container images through GitHub Actions (GHA) to GitHub Container Registry (GHCR).
+name: OCI for server
+
+on:
+  pull_request: ~
+  push:
+    tags:
+      - '*.*.*'
+
+  # Produce a nightly image every day at 6 a.m. CEST.
+  schedule:
+    - cron: '0 4 * * *'
+
+  # Allow job to be triggered manually.
+  workflow_dispatch:
+
+# Cancel in-progress jobs when pushing to the same branch.
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+# The name for the produced image at ghcr.io.
+env:
+  IMAGE_NAME: "${{ github.repository }}"
+  RECIPE_PATH: "release/oci-server"
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Acquire sources
+        uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+          cache: "pip"
+          cache-dependency-path: "pyproject.toml"
+
+      - name: Build wheel package
+        run: |
+          pip install build
+          python -m build
+
+      - name: Upload wheel package
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ runner.os }}-wheel-${{ github.sha }}
+          path: dist/*.whl
+          retention-days: 7
+
+      - name: Run tests
+        run: |
+          if [[ -f ${{ env.RECIPE_PATH }}/test.yml ]]; then
+            export DOCKER_BUILDKIT=1
+            export COMPOSE_DOCKER_CLI_BUILD=1
+            docker-compose --file ${{ env.RECIPE_PATH }}/test.yml build
+            docker-compose --file ${{ env.RECIPE_PATH }}/test.yml run sut
+          fi
+
+  oci:
+    needs: build_and_test
+    runs-on: ubuntu-latest
+    if: ${{ ! (startsWith(github.actor, 'dependabot') || github.event.pull_request.head.repo.fork ) }}
+
+    steps:
+      - name: Acquire sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Define image name and tags
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          # List of OCI images to use as base name for tags
+          images: |
+            ghcr.io/${{ env.IMAGE_NAME }}
+          # Generate OCI image tags based on the following events/attributes
+          tags: |
+            type=schedule,pattern=nightly
+            type=ref,event=pr
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Inspect metadata
+        run: |
+          echo "Tags:      ${{ steps.meta.outputs.tags }}"
+          echo "Labels:    ${{ steps.meta.outputs.labels }}"
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Cache OCI layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Inspect builder
+        run: |
+          echo "Name:      ${{ steps.buildx.outputs.name }}"
+          echo "Endpoint:  ${{ steps.buildx.outputs.endpoint }}"
+          echo "Status:    ${{ steps.buildx.outputs.status }}"
+          echo "Flags:     ${{ steps.buildx.outputs.flags }}"
+          echo "Platforms: ${{ steps.buildx.outputs.platforms }}"
+
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ github.token }}
+
+      - name: Build and push image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ${{ env.RECIPE_PATH }}/Dockerfile
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+      - name: Display git status
+        run: |
+          set -x
+          git describe --tags --always
+          git status
diff --git a/release/oci-examples/Dockerfile b/release/oci-examples/Dockerfile
@@ -0,0 +1,43 @@
+# syntax=docker/dockerfile:1
+
+# Use BuildKit's build-time cache mounts, it makes a huge difference on rebuilds.
+# - https://vsupalov.com/buildkit-cache-mount-dockerfile/
+# - https://github.com/FernandoMiguel/Buildkit#mounttypecache
+
+FROM python:3.11-slim-bullseye
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV TERM linux
+
+# Install Git, it is needed for `versioningit`.
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
+# Install distribution packages, with caching.
+RUN --mount=type=cache,id=apt,target=/var/cache/apt --mount=type=cache,id=apt,target=/var/lib/apt \
+    true \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests --yes git
+
+# Copy sources
+COPY . /src
+
+# Install package, with caching of dependency packages.
+RUN --mount=type=cache,id=pip,target=/root/.cache/pip \
+    pip install --use-pep517 --prefer-binary '/src[examples]'
+
+# Uninstall Git again.
+RUN apt-get --yes remove --purge git && apt-get --yes autoremove
+
+# Purge /src and /tmp directories.
+RUN rm -rf /src /tmp/*
+
+# Copy selftest.sh to the image
+COPY release/oci-examples/selftest.sh /usr/local/bin
+
+# Run program once, in order to let Python compile its files.
+RUN mlflow-cratedb --version
+RUN mlflow-cratedb cratedb --version
+
+# Copy example programs and documentation to the image.
+COPY examples /opt/ml-examples/bin
+COPY docs /opt/ml-examples/docs
diff --git a/release/oci-examples/selftest.sh b/release/oci-examples/selftest.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Fail on error.
+set -e
+
+# Display all commands.
+# set -x
+
+echo "Invoking MLflow adapter for CrateDB"
+mflow-cratedb --version
+mflow-cratedb cratedb --version
diff --git a/release/oci-examples/test.yml b/release/oci-examples/test.yml
@@ -0,0 +1,4 @@
+sut:
+  build: ../..
+  dockerfile: release/oci-examples/Dockerfile
+  command: selftest.sh
diff --git a/release/oci-server/Dockerfile b/release/oci-server/Dockerfile
@@ -0,0 +1,39 @@
+# syntax=docker/dockerfile:1
+
+# Use BuildKit's build-time cache mounts, it makes a huge difference on rebuilds.
+# - https://vsupalov.com/buildkit-cache-mount-dockerfile/
+# - https://github.com/FernandoMiguel/Buildkit#mounttypecache
+
+FROM python:3.11-slim-bullseye
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV TERM linux
+
+# Install Git, it is needed for `versioningit`.
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
+# Install distribution packages, with caching.
+RUN --mount=type=cache,id=apt,target=/var/cache/apt --mount=type=cache,id=apt,target=/var/lib/apt \
+    true \
+    && apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests --yes git
+
+# Copy sources
+COPY . /src
+
+# Install package, with caching of dependency packages.
+RUN --mount=type=cache,id=pip,target=/root/.cache/pip \
+    pip install --use-pep517 --prefer-binary '/src'
+
+# Uninstall Git again.
+RUN apt-get --yes remove --purge git && apt-get --yes autoremove
+
+# Purge /src and /tmp directories.
+RUN rm -rf /src /tmp/*
+
+# Copy selftest.sh to the image
+COPY release/oci-server/selftest.sh /usr/local/bin
+
+# Run program once, in order to let Python compile its files.
+RUN mlflow-cratedb --version
+RUN mlflow-cratedb cratedb --version