From 2eb9662527f862d74189ea87f4c171b28bdfc974 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Sun, 28 Jan 2024 09:41:01 -0600 Subject: [PATCH] Add multi-stage builds to our containers This change updates the builds for the containers to use a multistage build where applicable. All containers will now also cleanup the pycache files when detected, ensuring that we're building efficient images. Signed-off-by: Kevin Carter --- .github/workflows/pre-commit.yaml | 36 +++++++++++++++++++ .github/workflows/release-keystone-rxt.yml | 9 +++-- .github/workflows/release-nova-uefi.yml | 9 +++-- .github/workflows/release-octavia-ovn.yml | 7 ++-- .github/workflows/smoke-horizon-rxt.yml | 42 ++++++++++++++++++++++ .github/workflows/smoke-keystone-rxt.yml | 41 +++++++++++++++++++++ .github/workflows/smoke-nova-uefi.yml | 41 +++++++++++++++++++++ .github/workflows/smoke-octavia-ovn.yml | 42 ++++++++++++++++++++++ .pre-commit-config.yaml | 33 +++++++++++++++++ Containerfiles/HorizonRXT-Containerfile | 1 + Containerfiles/KeystoneRXT-Containerfile | 7 +++- Containerfiles/NovaEFI-Containerfile | 8 ++--- Containerfiles/OctaviaOVN-Containerfile | 7 +++- requirements.yml | 1 + 14 files changed, 270 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .github/workflows/smoke-horizon-rxt.yml create mode 100644 .github/workflows/smoke-keystone-rxt.yml create mode 100644 .github/workflows/smoke-nova-uefi.yml create mode 100644 .github/workflows/smoke-octavia-ovn.yml create mode 100644 .pre-commit-config.yaml create mode 120000 requirements.yml diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 00000000..b3063c40 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,36 @@ +name: Run pull-request syntax workflows +on: + - pull_request +jobs: + pre_commit: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.10" + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v23.1 + - name: Run Check + uses: pre-commit/action@v3.0.0 + with: + extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }} + - name: Checks failed, notification + if: failure() + run: | + echo "Tests Failed" + echo "Run the following command to identify issues" + echo "pre-commit run --files ${{ steps.changed-files.outputs.all_changed_files }}" + - name: Upload log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: pre-commit-py${{ matrix.python-version }} + path: /home/runner/.cache/pre-commit/pre-commit.log diff --git a/.github/workflows/release-keystone-rxt.yml b/.github/workflows/release-keystone-rxt.yml index e0ff9c69..e418ce6c 100644 --- a/.github/workflows/release-keystone-rxt.yml +++ b/.github/workflows/release-keystone-rxt.yml @@ -11,9 +11,9 @@ on: default: 'master-ubuntu_jammy' type: choice options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy + - master-ubuntu_jammy + - 2023.1-ubuntu_jammy + - 2023.2-ubuntu_jammy # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -47,6 +47,8 @@ jobs: # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Dynamically set MY_DATE environment variable + run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - name: Build and push Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: @@ -55,6 +57,7 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} labels: ${{ steps.meta.outputs.labels }} build-args: | VERSION=${{ github.event.inputs.imageTag }} diff --git a/.github/workflows/release-nova-uefi.yml b/.github/workflows/release-nova-uefi.yml index bdc15b7d..a4e81060 100644 --- a/.github/workflows/release-nova-uefi.yml +++ b/.github/workflows/release-nova-uefi.yml @@ -11,9 +11,9 @@ on: default: 'master-ubuntu_jammy' type: choice options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy + - master-ubuntu_jammy + - 2023.1-ubuntu_jammy + - 2023.2-ubuntu_jammy # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -47,6 +47,8 @@ jobs: # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Dynamically set MY_DATE environment variable + run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - name: Build and push Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: @@ -55,6 +57,7 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} labels: ${{ steps.meta.outputs.labels }} build-args: | VERSION=${{ github.event.inputs.imageTag }} diff --git a/.github/workflows/release-octavia-ovn.yml b/.github/workflows/release-octavia-ovn.yml index 2551d4b0..e29137fc 100644 --- a/.github/workflows/release-octavia-ovn.yml +++ b/.github/workflows/release-octavia-ovn.yml @@ -11,14 +11,14 @@ on: default: 'master-ubuntu_jammy' type: choice options: - - master-ubuntu_jammy + - master-ubuntu_jammy pluginTag: description: 'Set plugin version' required: true default: '5.0.0' type: choice options: - - '5.0.0' + - '5.0.0' # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: REGISTRY: ghcr.io @@ -51,6 +51,8 @@ jobs: # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Dynamically set MY_DATE environment variable + run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - name: Build and push Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: @@ -59,6 +61,7 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} labels: ${{ steps.meta.outputs.labels }} build-args: | VERSION=${{ github.event.inputs.imageTag }} diff --git a/.github/workflows/smoke-horizon-rxt.yml b/.github/workflows/smoke-horizon-rxt.yml new file mode 100644 index 00000000..e81c1c69 --- /dev/null +++ b/.github/workflows/smoke-horizon-rxt.yml @@ -0,0 +1,42 @@ +# +name: Create and publish a the Horizon RXT compatible image + +on: + pull_request: + paths: + - Containerfiles/HorizonRXT-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/HorizonRXT-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon-rxt:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy + PLUGIN_VERSION=master diff --git a/.github/workflows/smoke-keystone-rxt.yml b/.github/workflows/smoke-keystone-rxt.yml new file mode 100644 index 00000000..1687a05f --- /dev/null +++ b/.github/workflows/smoke-keystone-rxt.yml @@ -0,0 +1,41 @@ +# +name: Create and publish a the Keystone RXT compatible image + +on: + pull_request: + paths: + - Containerfiles/KeystoneRXT-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/KeystoneRXT-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy diff --git a/.github/workflows/smoke-nova-uefi.yml b/.github/workflows/smoke-nova-uefi.yml new file mode 100644 index 00000000..ea8f37da --- /dev/null +++ b/.github/workflows/smoke-nova-uefi.yml @@ -0,0 +1,41 @@ +# +name: Create and publish a the Nova EFI compatible image + +on: + pull_request: + paths: + - Containerfiles/NovaEFI-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/NovaEFI-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy diff --git a/.github/workflows/smoke-octavia-ovn.yml b/.github/workflows/smoke-octavia-ovn.yml new file mode 100644 index 00000000..aabafd30 --- /dev/null +++ b/.github/workflows/smoke-octavia-ovn.yml @@ -0,0 +1,42 @@ +# +name: Create and publish a the Octavia OVN compatible image + +on: + pull_request: + paths: + - Containerfiles/OctaviaOVN-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/OctaviaOVN-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy + PLUGIN_VERSION='5.0.0' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..59e231c7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + - id: check-byte-order-marker + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + files: .*\.(yaml|yml)$ + args: + - "--allow-multiple-documents" + - "--unsafe" + - id: debug-statements + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/ansible-community/ansible-lint + rev: v5.3.2 + hooks: + - id: ansible-lint + additional_dependencies: + - ansible-core + - yamllint + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 2.1.6 + hooks: + - id: shellcheck diff --git a/Containerfiles/HorizonRXT-Containerfile b/Containerfiles/HorizonRXT-Containerfile index 3fc30a77..696b2c97 100644 --- a/Containerfiles/HorizonRXT-Containerfile +++ b/Containerfiles/HorizonRXT-Containerfile @@ -10,5 +10,6 @@ RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ git+https://opendev.org/openstack/heat-dashboard@${PLUGIN_VERSION}#egg=heat_dashboard \ git+https://opendev.org/openstack/octavia-dashboard@${PLUGIN_VERSION}#egg=octavia_dashboard RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + FROM openstackhelm/horizon:${VERSION} COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/KeystoneRXT-Containerfile b/Containerfiles/KeystoneRXT-Containerfile index a0d35ac5..43da64bf 100644 --- a/Containerfiles/KeystoneRXT-Containerfile +++ b/Containerfiles/KeystoneRXT-Containerfile @@ -1,3 +1,8 @@ ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/keystone:$VERSION +FROM openstackhelm/keystone:${VERSION} as build +RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip RUN /var/lib/openstack/bin/pip install keystone-rxt +RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + +FROM openstackhelm/keystone:${VERSION} +COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/NovaEFI-Containerfile b/Containerfiles/NovaEFI-Containerfile index e8db4abb..c88c2e5b 100644 --- a/Containerfiles/NovaEFI-Containerfile +++ b/Containerfiles/NovaEFI-Containerfile @@ -3,10 +3,10 @@ FROM openstackhelm/nova:$VERSION # Packages for the following features: # - Nova: EFI # - Nova: iSCSI +# Py Packages for the following features: +# - Nova: Libosinfo RUN apt update && apt install -y ovmf open-iscsi multipath-tools libgirepository-1.0-1 libgirepository1.0-dev \ libcairo2-dev python3-dev gcc libosinfo-bin gir1.2-libosinfo-1.0; \ rm -rf /var/cache/apt/archives /var/lib/apt/lists; \ - apt clean -# Packages for the following features: -# - Nova: Libosinfo -RUN /var/lib/openstack/bin/pip install pygobject + apt clean; /var/lib/openstack/bin/pip install pygobject; \ + find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete diff --git a/Containerfiles/OctaviaOVN-Containerfile b/Containerfiles/OctaviaOVN-Containerfile index 3ce7a8f8..1661e010 100644 --- a/Containerfiles/OctaviaOVN-Containerfile +++ b/Containerfiles/OctaviaOVN-Containerfile @@ -1,4 +1,9 @@ ARG VERSION=master-ubuntu_jammy -FROM docker.io/loci/octavia:${VERSION} +FROM docker.io/loci/octavia:${VERSION} as build ARG PLUGIN_VERSION=5.0.0 +RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip RUN /var/lib/openstack/bin/pip install ovn-octavia-provider===${PLUGIN_VERSION} +RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + +FROM docker.io/loci/octavia:${VERSION} +COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/requirements.yml b/requirements.yml new file mode 120000 index 00000000..59b55f9b --- /dev/null +++ b/requirements.yml @@ -0,0 +1 @@ +ansible-collection-requirements.yml \ No newline at end of file