Add multi-stage builds to our containers

This change updates the builds for the containers to use a multistage build where applicable. All containers will now also cleanup the pycache files when detected, ensuring that we're building efficient images. Signed-off-by: Kevin Carter <[email protected]>
rackerlabs · Jan 28, 2024 · 2eb9662 · 2eb9662
1 parent c2c78a8
commit 2eb9662
Show file tree

Hide file tree

Showing 14 changed files with 270 additions and 14 deletions.
diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
@@ -0,0 +1,36 @@
+name: Run pull-request syntax workflows
+on:
+  - pull_request
+jobs:
+  pre_commit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+      - name: Run Check
+        uses: pre-commit/[email protected]
+        with:
+          extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }}
+      - name: Checks failed, notification
+        if: failure()
+        run: |
+          echo "Tests Failed"
+          echo "Run the following command to identify issues"
+          echo "pre-commit run --files ${{ steps.changed-files.outputs.all_changed_files }}"
+      - name: Upload log artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: pre-commit-py${{ matrix.python-version }}
+          path: /home/runner/.cache/pre-commit/pre-commit.log
diff --git a/.github/workflows/release-keystone-rxt.yml b/.github/workflows/release-keystone-rxt.yml
@@ -11,9 +11,9 @@ on:
         default: 'master-ubuntu_jammy'
         type: choice
         options:
-        - master-ubuntu_jammy
-        - 2023.1-ubuntu_jammy
-        - 2023.2-ubuntu_jammy
+          - master-ubuntu_jammy
+          - 2023.1-ubuntu_jammy
+          - 2023.2-ubuntu_jammy
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -47,6 +47,8 @@ jobs:
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Dynamically set MY_DATE environment variable
+        run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV
       - name: Build and push Docker image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
@@ -55,6 +57,7 @@ jobs:
           push: true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             VERSION=${{ github.event.inputs.imageTag }}
diff --git a/.github/workflows/release-nova-uefi.yml b/.github/workflows/release-nova-uefi.yml
@@ -11,9 +11,9 @@ on:
         default: 'master-ubuntu_jammy'
         type: choice
         options:
-        - master-ubuntu_jammy
-        - 2023.1-ubuntu_jammy
-        - 2023.2-ubuntu_jammy
+          - master-ubuntu_jammy
+          - 2023.1-ubuntu_jammy
+          - 2023.2-ubuntu_jammy
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -47,6 +47,8 @@ jobs:
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Dynamically set MY_DATE environment variable
+        run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV
       - name: Build and push Docker image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
@@ -55,6 +57,7 @@ jobs:
           push: true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             VERSION=${{ github.event.inputs.imageTag }}
diff --git a/.github/workflows/release-octavia-ovn.yml b/.github/workflows/release-octavia-ovn.yml
@@ -11,14 +11,14 @@ on:
         default: 'master-ubuntu_jammy'
         type: choice
         options:
-        - master-ubuntu_jammy
+          - master-ubuntu_jammy
       pluginTag:
         description: 'Set plugin version'
         required: true
         default: '5.0.0'
         type: choice
         options:
-        - '5.0.0'
+          - '5.0.0'
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
   REGISTRY: ghcr.io
@@ -51,6 +51,8 @@ jobs:
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Dynamically set MY_DATE environment variable
+        run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV
       - name: Build and push Docker image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
@@ -59,6 +61,7 @@ jobs:
           push: true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             VERSION=${{ github.event.inputs.imageTag }}

diff --git a/.github/workflows/smoke-horizon-rxt.yml b/.github/workflows/smoke-horizon-rxt.yml
@@ -0,0 +1,42 @@
+#
+name: Create and publish a the Horizon RXT compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/HorizonRXT-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/HorizonRXT-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon-rxt:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
+            PLUGIN_VERSION=master
diff --git a/.github/workflows/smoke-keystone-rxt.yml b/.github/workflows/smoke-keystone-rxt.yml
@@ -0,0 +1,41 @@
+#
+name: Create and publish a the Keystone RXT compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/KeystoneRXT-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/KeystoneRXT-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
diff --git a/.github/workflows/smoke-nova-uefi.yml b/.github/workflows/smoke-nova-uefi.yml
@@ -0,0 +1,41 @@
+#
+name: Create and publish a the Nova EFI compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/NovaEFI-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/NovaEFI-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
diff --git a/.github/workflows/smoke-octavia-ovn.yml b/.github/workflows/smoke-octavia-ovn.yml
@@ -0,0 +1,42 @@
+#
+name: Create and publish a the Octavia OVN compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/OctaviaOVN-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/OctaviaOVN-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
+            PLUGIN_VERSION='5.0.0'
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,33 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+    - id: end-of-file-fixer
+    - id: trailing-whitespace
+    - id: mixed-line-ending
+    - id: check-byte-order-marker
+    - id: check-executables-have-shebangs
+    - id: check-merge-conflict
+    - id: check-symlinks
+    - id: check-yaml
+      files: .*\.(yaml|yml)$
+      args:
+      - "--allow-multiple-documents"
+      - "--unsafe"
+    - id: debug-statements
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+    - id: black
+  - repo: https://github.com/ansible-community/ansible-lint
+    rev: v5.3.2
+    hooks:
+    - id: ansible-lint
+      additional_dependencies:
+      - ansible-core
+      - yamllint
+  - repo: https://github.com/jumanjihouse/pre-commit-hooks
+    rev: 2.1.6
+    hooks:
+    - id: shellcheck
diff --git a/Containerfiles/HorizonRXT-Containerfile b/Containerfiles/HorizonRXT-Containerfile
@@ -10,5 +10,6 @@ RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \
                                git+https://opendev.org/openstack/heat-dashboard@${PLUGIN_VERSION}#egg=heat_dashboard \
                                git+https://opendev.org/openstack/octavia-dashboard@${PLUGIN_VERSION}#egg=octavia_dashboard
 RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
+
 FROM openstackhelm/horizon:${VERSION}
 COPY --from=build /var/lib/openstack/. /var/lib/openstack/
diff --git a/Containerfiles/KeystoneRXT-Containerfile b/Containerfiles/KeystoneRXT-Containerfile
@@ -1,3 +1,8 @@
 ARG VERSION=master-ubuntu_jammy
-FROM openstackhelm/keystone:$VERSION
+FROM openstackhelm/keystone:${VERSION} as build
+RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip
 RUN /var/lib/openstack/bin/pip install keystone-rxt
+RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
+
+FROM openstackhelm/keystone:${VERSION}
+COPY --from=build /var/lib/openstack/. /var/lib/openstack/
diff --git a/Containerfiles/NovaEFI-Containerfile b/Containerfiles/NovaEFI-Containerfile
@@ -3,10 +3,10 @@ FROM openstackhelm/nova:$VERSION
 # Packages for the following features:
 # - Nova: EFI
 # - Nova: iSCSI
+# Py Packages for the following features:
+# - Nova: Libosinfo
 RUN apt update && apt install -y ovmf open-iscsi multipath-tools libgirepository-1.0-1 libgirepository1.0-dev \
                                  libcairo2-dev python3-dev gcc libosinfo-bin gir1.2-libosinfo-1.0; \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists; \
-    apt clean
-# Packages for the following features:
-# - Nova: Libosinfo
-RUN /var/lib/openstack/bin/pip install pygobject
+    apt clean; /var/lib/openstack/bin/pip install pygobject; \
+    find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
diff --git a/Containerfiles/OctaviaOVN-Containerfile b/Containerfiles/OctaviaOVN-Containerfile
@@ -1,4 +1,9 @@
 ARG VERSION=master-ubuntu_jammy
-FROM docker.io/loci/octavia:${VERSION}
+FROM docker.io/loci/octavia:${VERSION} as build
 ARG PLUGIN_VERSION=5.0.0
+RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip
 RUN /var/lib/openstack/bin/pip install ovn-octavia-provider===${PLUGIN_VERSION}
+RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
+
+FROM docker.io/loci/octavia:${VERSION}
+COPY --from=build /var/lib/openstack/. /var/lib/openstack/
diff --git a/requirements.yml b/requirements.yml
@@ -0,0 +1 @@
+ansible-collection-requirements.yml