Simplify image builds by using QEMU to do multi-arch builds, but sinc…

…e they are slow, only do the multi-arch builds for releases. Also support passing cargo build args to the Docker image builder, allowing us to enable all features. Signed-off-by: Hiram Chirino <[email protected]>
Kuadrant · Jun 3, 2024 · 6df0c4d · 6df0c4d
1 parent 4b2dc62
commit 6df0c4d
Show file tree

Hide file tree

Showing 6 changed files with 135 additions and 149 deletions.
diff --git a/.github/actions/build-image/action.yaml b/.github/actions/build-image/action.yaml
@@ -0,0 +1,55 @@
+name: 'Container Image Build'
+description: 'Builds a container image for the project'
+inputs:
+  platforms:
+    description: 'comma seperated list of platforms to build the image for'
+    required: true
+    default: 'linux/amd64'
+runs:
+  using: "composite"
+  steps:
+
+    - name: Set up QEMU for Multi-Arch Builds
+      uses: docker/setup-qemu-action@v3
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: |
+          ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador
+      tags: |
+        # SHA tag for main branch
+        type=raw,value=${{ github.sha }},enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }}
+        # set latest tag for main branch
+        type=raw,value=latest,enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }}
+        # set ref name tag for non-main branches
+        type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != env.MAIN_BRANCH_NAME }}
+
+    - name: Login to container registry
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.IMG_REGISTRY_USERNAME }}
+        password: ${{ secrets.IMG_REGISTRY_TOKEN }}
+        registry: ${{ env.IMG_REGISTRY_HOST }}
+
+    - name: Build and Push
+      id: build
+      uses: docker/build-push-action@v5
+      with:
+        platforms: ${{ inputs.platforms }}
+        push: true
+        file: ./Dockerfile
+        tags: |
+          ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ steps.meta.outputs.version }}
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          GITHUB_SHA=${{ github.sha }}
+
+        context: .
+        labels: ${{ steps.meta.outputs.labels }}
+        outputs: type=image,name=${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador,push-by-digest=true,name-canonical=true,push=true
+        provenance: false
diff --git a/.github/workflows/build-image.yaml b/.github/workflows/build-image.yaml
@@ -6,8 +6,6 @@ on:
   push:
     branches:
       - main
-    tags:
-      - "*"
 
 env:
   IMG_REGISTRY_HOST: quay.io
@@ -20,97 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - dockerfile: Dockerfile
-            platform: linux/amd64
-            scope: build-amd
-          - dockerfile: Dockerfile.aarch64
-            platform: linux/arm64
-            scope: build-arm
     steps:
       - name: Check out code
         uses: actions/checkout@v4
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador
-      - name: Login to container registry
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.IMG_REGISTRY_USERNAME }}
-          password: ${{ secrets.IMG_REGISTRY_TOKEN }}
-          registry: ${{ env.IMG_REGISTRY_HOST }}
-      - name: Build and push by digest
-        id: build
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          labels: ${{ steps.meta.outputs.labels }}
-          build-args: |
-            GITHUB_SHA=${{ github.sha }}
-          cache-from: type=gha,scope=${{ matrix.scope }}
-          cache-to: type=gha,mode=max,scope=${{ matrix.scope}}
-          outputs: type=image,name=${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador,push-by-digest=true,name-canonical=true,push=true
-          file: ${{ matrix.dockerfile }}
-          platforms: |
-            ${{ matrix.platform }}
-          provenance: false
-      - name: Export digest
-        run: |
-          mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
-          touch "/tmp/digests/${digest#sha256:}"
-      - name: Upload digest
-        uses: actions/upload-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-  merge:
-    runs-on: ubuntu-latest
-    needs:
-      - build
-    steps:
-      - name: Download digests
-        uses: actions/download-artifact@v3
-        with:
-          name: digests
-          path: /tmp/digests
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: |
-            ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador
-          tags: |
-            # SHA tag for main branch
-            type=raw,value=${{ github.sha }},enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }}
-            # set latest tag for main branch
-            type=raw,value=latest,enable=${{ github.ref_name == env.MAIN_BRANCH_NAME }}
-            # set ref name tag for non-main branches
-            type=raw,value=${{ github.ref_name }},enable=${{ github.ref_name != env.MAIN_BRANCH_NAME }}
-      - name: Login to container registry
-        uses: docker/login-action@v2
+
+      - name: Build/Push the Container Image
+        uses: ./.github/actions/build-image
         with:
-          username: ${{ secrets.IMG_REGISTRY_USERNAME }}
-          password: ${{ secrets.IMG_REGISTRY_TOKEN }}
-          registry: ${{ env.IMG_REGISTRY_HOST }}
-      - name: Create manifest list and push
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-            $(printf '${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador@sha256:%s ' *)
-      - name: Inspect image
-        run: |
-          docker buildx imagetools inspect ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ steps.meta.outputs.version }}
-      - name: Smoke Test
-        run: |
-          docker run --rm -t ${{ env.IMG_REGISTRY_HOST }}/${{ env.IMG_REGISTRY_ORG }}/limitador:${{ steps.meta.outputs.version }} limitador-server --help
+          platforms: linux/amd64
diff --git a/.github/workflows/build-multi-arch-image.yaml b/.github/workflows/build-multi-arch-image.yaml
@@ -0,0 +1,28 @@
+---
+name: Build Image
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - "*"
+
+env:
+  IMG_REGISTRY_HOST: quay.io
+  IMG_REGISTRY_ORG: kuadrant
+  MAIN_BRANCH_NAME: main
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Build/Push the Container Image
+        uses: ./.github/actions/build-image
+        with:
+          platforms: linux/amd64,linux/arm64
diff --git a/Dockerfile b/Dockerfile
@@ -4,20 +4,32 @@
 
 # Use bullseye as build image instead of Bookworm as ubi9 does not not have GLIBCXX_3.4.30
 # https://access.redhat.com/solutions/6969351
-FROM --platform=${BUILDPLATFORM} rust:1.78.0-bullseye as limitador-build
+FROM rust:1.78.0-bullseye as limitador-build
 
 RUN apt update && apt upgrade -y \
     && apt install -y protobuf-compiler clang
 
 WORKDIR /usr/src/limitador
 
 ARG GITHUB_SHA
+ARG CARGO_ARGS
 ENV GITHUB_SHA=${GITHUB_SHA:-unknown}
 ENV RUSTFLAGS="-C target-feature=-crt-static"
 
-COPY . .
+# We set the env here just to make sure that the build is invalidated if the args change
+ENV CARGO_ARGS=${CARGO_ARGS}
 
-RUN cargo build --release
+# The following allows us to cache the Cargo dependency downloads with image layers
+COPY Cargo.toml Cargo.lock ./
+COPY limitador/Cargo.toml ./limitador/
+COPY limitador-server/Cargo.toml ./limitador-server/
+RUN mkdir -p limitador-server/src && echo 'fn main() {}' > limitador-server/src/main.rs
+RUN cargo build --release ${CARGO_ARGS}
+
+COPY ./limitador ./limitador
+COPY ./limitador-server ./limitador-server
+
+RUN cargo build --release ${CARGO_ARGS}
 
 # ------------------------------------------------------------------------------
 # Run Stage

diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64
diff --git a/limitador-server/sandbox/docker-compose-limitador-distributed.yaml b/limitador-server/sandbox/docker-compose-limitador-distributed.yaml
@@ -0,0 +1,33 @@
+---
+version: '3.8'
+services:
+  limitador:
+    image: ${LIMITADOR_IMAGE:-limitador-testing}
+    depends_on:
+      - envoy
+      - redis
+    command:
+      - limitador-server
+      - --rls-ip
+      - 0.0.0.0
+      - --rls-port
+      - "8081"
+      - --http-ip
+      - 0.0.0.0
+      - --http-port
+      - "8080"
+      - -vvv
+      - --grpc-reflection-service
+      - /opt/kuadrant/limits/limits.yaml
+      - redis
+      - redis://redis:6379
+    expose:
+      - "8080"
+      - "8081"
+    ports:
+      - "18080:8080"
+      - "18081:8081"
+    volumes:
+      - ./limits.yaml:/opt/kuadrant/limits/limits.yaml
+  redis:
+    image: redis:5