Add multi-stage builds to our containers

This change updates the builds for the containers to use a multistage build where applicable. All containers will now also cleanup the pycache files when detected, ensuring that we're building efficient images. With this change we'll now publish container images with an epoc so that we can use point in time releases when needed. This change adds pre-commit hooks to our setup so that we can begin running some static analysis against our code base. > To ensure that our tests run we've made a change to the `addons.yml` file to correct a syntax error. Ansible-lint config has been added to ignore most of the infractions at this point, we'll need to address those rule violations as we develop the tool-chain. Signed-off-by: Kevin Carter <[email protected]>
rackerlabs · Jan 28, 2024 · 942574b · 942574b
1 parent c2c78a8
commit 942574b
Show file tree

Hide file tree

Showing 16 changed files with 282 additions and 16 deletions.
diff --git a/.ansible-lint b/.ansible-lint
@@ -0,0 +1,9 @@
+skip_list:
+- yaml
+- jinja
+- no-free-form
+- name
+- var-naming
+- risky-file-permissions
+- no-changed-when
+- fqcn
diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml
@@ -0,0 +1,36 @@
+name: Run pull-request syntax workflows
+on:
+  - pull_request
+jobs:
+  pre_commit:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.10"
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+      - name: Run Check
+        uses: pre-commit/[email protected]
+        with:
+          extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }}
+      - name: Checks failed, notification
+        if: failure()
+        run: |
+          echo "Tests Failed"
+          echo "Run the following command to identify issues"
+          echo "pre-commit run --files ${{ steps.changed-files.outputs.all_changed_files }}"
+      - name: Upload log artifacts on failure
+        if: failure()
+        uses: actions/upload-artifact@v2
+        with:
+          name: pre-commit-py${{ matrix.python-version }}
+          path: /home/runner/.cache/pre-commit/pre-commit.log
diff --git a/.github/workflows/release-keystone-rxt.yml b/.github/workflows/release-keystone-rxt.yml
@@ -11,9 +11,9 @@ on:
         default: 'master-ubuntu_jammy'
         type: choice
         options:
-        - master-ubuntu_jammy
-        - 2023.1-ubuntu_jammy
-        - 2023.2-ubuntu_jammy
+          - master-ubuntu_jammy
+          - 2023.1-ubuntu_jammy
+          - 2023.2-ubuntu_jammy
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -47,6 +47,8 @@ jobs:
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Dynamically set MY_DATE environment variable
+        run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV
       - name: Build and push Docker image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
@@ -55,6 +57,7 @@ jobs:
           push: true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             VERSION=${{ github.event.inputs.imageTag }}
diff --git a/.github/workflows/release-nova-uefi.yml b/.github/workflows/release-nova-uefi.yml
@@ -11,9 +11,9 @@ on:
         default: 'master-ubuntu_jammy'
         type: choice
         options:
-        - master-ubuntu_jammy
-        - 2023.1-ubuntu_jammy
-        - 2023.2-ubuntu_jammy
+          - master-ubuntu_jammy
+          - 2023.1-ubuntu_jammy
+          - 2023.2-ubuntu_jammy
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -47,6 +47,8 @@ jobs:
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Dynamically set MY_DATE environment variable
+        run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV
       - name: Build and push Docker image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
@@ -55,6 +57,7 @@ jobs:
           push: true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             VERSION=${{ github.event.inputs.imageTag }}
diff --git a/.github/workflows/release-octavia-ovn.yml b/.github/workflows/release-octavia-ovn.yml
@@ -11,14 +11,15 @@ on:
         default: 'master-ubuntu_jammy'
         type: choice
         options:
-        - master-ubuntu_jammy
+          - master-ubuntu_jammy
       pluginTag:
         description: 'Set plugin version'
         required: true
         default: '5.0.0'
         type: choice
         options:
-        - '5.0.0'
+          - '5.0.0'
+
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
   REGISTRY: ghcr.io
@@ -51,6 +52,8 @@ jobs:
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.
+      - name: Dynamically set MY_DATE environment variable
+        run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV
       - name: Build and push Docker image
         uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
         with:
@@ -59,6 +62,7 @@ jobs:
           push: true
           tags: |
             ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }}
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }}
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             VERSION=${{ github.event.inputs.imageTag }}

diff --git a/.github/workflows/smoke-horizon-rxt.yml b/.github/workflows/smoke-horizon-rxt.yml
@@ -0,0 +1,42 @@
+#
+name: Run build check for the Horizon RXT compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/HorizonRXT-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/HorizonRXT-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon-rxt:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
+            PLUGIN_VERSION=master
diff --git a/.github/workflows/smoke-keystone-rxt.yml b/.github/workflows/smoke-keystone-rxt.yml
@@ -0,0 +1,41 @@
+#
+name: Run build check for the Keystone RXT compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/KeystoneRXT-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/KeystoneRXT-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
diff --git a/.github/workflows/smoke-nova-uefi.yml b/.github/workflows/smoke-nova-uefi.yml
@@ -0,0 +1,41 @@
+#
+name: Run build check for the Nova EFI compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/NovaEFI-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/NovaEFI-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
diff --git a/.github/workflows/smoke-octavia-ovn.yml b/.github/workflows/smoke-octavia-ovn.yml
@@ -0,0 +1,42 @@
+#
+name: Run build check for the Octavia OVN compatible image
+
+on:
+  pull_request:
+    paths:
+      - Containerfiles/OctaviaOVN-Containerfile
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          file: Containerfiles/OctaviaOVN-Containerfile
+          push: false
+          tags: |
+            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:master-ubuntu_jammy
+          build-args: |
+            VERSION=master-ubuntu_jammy
+            PLUGIN_VERSION=5.0.0
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,33 @@
+---
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    - id: end-of-file-fixer
+    - id: trailing-whitespace
+    - id: mixed-line-ending
+    - id: check-byte-order-marker
+    - id: check-executables-have-shebangs
+    - id: check-merge-conflict
+    - id: check-symlinks
+    - id: check-yaml
+      files: .*\.(yaml|yml)$
+      args:
+      - "--allow-multiple-documents"
+      - "--unsafe"
+    - id: debug-statements
+  - repo: https://github.com/psf/black
+    rev: 24.1.1
+    hooks:
+    - id: black
+  - repo: https://github.com/ansible-community/ansible-lint
+    rev: v6.22.2
+    hooks:
+    - id: ansible-lint
+      additional_dependencies:
+      - ansible
+      - yamllint
+  - repo: https://github.com/jumanjihouse/pre-commit-hooks
+    rev: 3.0.0
+    hooks:
+    - id: shellcheck
diff --git a/Containerfiles/HorizonRXT-Containerfile b/Containerfiles/HorizonRXT-Containerfile
@@ -10,5 +10,6 @@ RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \
                                git+https://opendev.org/openstack/heat-dashboard@${PLUGIN_VERSION}#egg=heat_dashboard \
                                git+https://opendev.org/openstack/octavia-dashboard@${PLUGIN_VERSION}#egg=octavia_dashboard
 RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
+
 FROM openstackhelm/horizon:${VERSION}
 COPY --from=build /var/lib/openstack/. /var/lib/openstack/
diff --git a/Containerfiles/KeystoneRXT-Containerfile b/Containerfiles/KeystoneRXT-Containerfile
@@ -1,3 +1,8 @@
 ARG VERSION=master-ubuntu_jammy
-FROM openstackhelm/keystone:$VERSION
+FROM openstackhelm/keystone:${VERSION} as build
+RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip
 RUN /var/lib/openstack/bin/pip install keystone-rxt
+RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
+
+FROM openstackhelm/keystone:${VERSION}
+COPY --from=build /var/lib/openstack/. /var/lib/openstack/
diff --git a/Containerfiles/NovaEFI-Containerfile b/Containerfiles/NovaEFI-Containerfile
@@ -3,10 +3,10 @@ FROM openstackhelm/nova:$VERSION
 # Packages for the following features:
 # - Nova: EFI
 # - Nova: iSCSI
+# Py Packages for the following features:
+# - Nova: Libosinfo
 RUN apt update && apt install -y ovmf open-iscsi multipath-tools libgirepository-1.0-1 libgirepository1.0-dev \
                                  libcairo2-dev python3-dev gcc libosinfo-bin gir1.2-libosinfo-1.0; \
     rm -rf /var/cache/apt/archives /var/lib/apt/lists; \
-    apt clean
-# Packages for the following features:
-# - Nova: Libosinfo
-RUN /var/lib/openstack/bin/pip install pygobject
+    apt clean; /var/lib/openstack/bin/pip install pygobject; \
+    find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
diff --git a/Containerfiles/OctaviaOVN-Containerfile b/Containerfiles/OctaviaOVN-Containerfile
@@ -1,4 +1,9 @@
 ARG VERSION=master-ubuntu_jammy
-FROM docker.io/loci/octavia:${VERSION}
+FROM docker.io/loci/octavia:${VERSION} as build
 ARG PLUGIN_VERSION=5.0.0
-RUN /var/lib/openstack/bin/pip install ovn-octavia-provider===${PLUGIN_VERSION}
+RUN /var/lib/openstack/bin/pip install --index-url https://pypi.python.org/simple --upgrade pip
+RUN /var/lib/openstack/bin/pip install --index-url https://pypi.python.org/simple ovn-octavia-provider==${PLUGIN_VERSION}
+RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete
+
+FROM docker.io/loci/octavia:${VERSION}
+COPY --from=build /var/lib/openstack/. /var/lib/openstack/
diff --git a/ansible/inventory/openstack-enterprise/group_vars/k8s_cluster/addons.yml b/ansible/inventory/openstack-enterprise/group_vars/k8s_cluster/addons.yml
@@ -176,7 +176,7 @@ metallb_speaker_enabled: "{{ metallb_enabled }}"
 metallb_protocol: "layer2"
 # metallb_port: "7472"
 # metallb_memberlist_port: "7946"
- metallb_config:
+metallb_config:
    speaker:
      nodeselector:
        kubernetes.io/os: "linux"