From 942574bc9ced61bdb7ebcd8337516db079320a25 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Sun, 28 Jan 2024 09:41:01 -0600 Subject: [PATCH] Add multi-stage builds to our containers This change updates the builds for the containers to use a multistage build where applicable. All containers will now also cleanup the pycache files when detected, ensuring that we're building efficient images. With this change we'll now publish container images with an epoc so that we can use point in time releases when needed. This change adds pre-commit hooks to our setup so that we can begin running some static analysis against our code base. > To ensure that our tests run we've made a change to the `addons.yml` file to correct a syntax error. Ansible-lint config has been added to ignore most of the infractions at this point, we'll need to address those rule violations as we develop the tool-chain. Signed-off-by: Kevin Carter --- .ansible-lint | 9 ++++ .github/workflows/pre-commit.yaml | 36 ++++++++++++++++ .github/workflows/release-keystone-rxt.yml | 9 ++-- .github/workflows/release-nova-uefi.yml | 9 ++-- .github/workflows/release-octavia-ovn.yml | 8 +++- .github/workflows/smoke-horizon-rxt.yml | 42 +++++++++++++++++++ .github/workflows/smoke-keystone-rxt.yml | 41 ++++++++++++++++++ .github/workflows/smoke-nova-uefi.yml | 41 ++++++++++++++++++ .github/workflows/smoke-octavia-ovn.yml | 42 +++++++++++++++++++ .pre-commit-config.yaml | 33 +++++++++++++++ Containerfiles/HorizonRXT-Containerfile | 1 + Containerfiles/KeystoneRXT-Containerfile | 7 +++- Containerfiles/NovaEFI-Containerfile | 8 ++-- Containerfiles/OctaviaOVN-Containerfile | 9 +++- .../group_vars/k8s_cluster/addons.yml | 2 +- requirements.yml | 1 + 16 files changed, 282 insertions(+), 16 deletions(-) create mode 100644 .ansible-lint create mode 100644 .github/workflows/pre-commit.yaml create mode 100644 .github/workflows/smoke-horizon-rxt.yml create mode 100644 .github/workflows/smoke-keystone-rxt.yml create mode 100644 .github/workflows/smoke-nova-uefi.yml create mode 100644 .github/workflows/smoke-octavia-ovn.yml create mode 100644 .pre-commit-config.yaml create mode 120000 requirements.yml diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 00000000..1138f52f --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,9 @@ +skip_list: +- yaml +- jinja +- no-free-form +- name +- var-naming +- risky-file-permissions +- no-changed-when +- fqcn diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 00000000..b3063c40 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,36 @@ +name: Run pull-request syntax workflows +on: + - pull_request +jobs: + pre_commit: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.10" + steps: + - name: Check out the repo + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v23.1 + - name: Run Check + uses: pre-commit/action@v3.0.0 + with: + extra_args: --files ${{ steps.changed-files.outputs.all_changed_files }} + - name: Checks failed, notification + if: failure() + run: | + echo "Tests Failed" + echo "Run the following command to identify issues" + echo "pre-commit run --files ${{ steps.changed-files.outputs.all_changed_files }}" + - name: Upload log artifacts on failure + if: failure() + uses: actions/upload-artifact@v2 + with: + name: pre-commit-py${{ matrix.python-version }} + path: /home/runner/.cache/pre-commit/pre-commit.log diff --git a/.github/workflows/release-keystone-rxt.yml b/.github/workflows/release-keystone-rxt.yml index e0ff9c69..e418ce6c 100644 --- a/.github/workflows/release-keystone-rxt.yml +++ b/.github/workflows/release-keystone-rxt.yml @@ -11,9 +11,9 @@ on: default: 'master-ubuntu_jammy' type: choice options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy + - master-ubuntu_jammy + - 2023.1-ubuntu_jammy + - 2023.2-ubuntu_jammy # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -47,6 +47,8 @@ jobs: # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Dynamically set MY_DATE environment variable + run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - name: Build and push Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: @@ -55,6 +57,7 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} labels: ${{ steps.meta.outputs.labels }} build-args: | VERSION=${{ github.event.inputs.imageTag }} diff --git a/.github/workflows/release-nova-uefi.yml b/.github/workflows/release-nova-uefi.yml index bdc15b7d..a4e81060 100644 --- a/.github/workflows/release-nova-uefi.yml +++ b/.github/workflows/release-nova-uefi.yml @@ -11,9 +11,9 @@ on: default: 'master-ubuntu_jammy' type: choice options: - - master-ubuntu_jammy - - 2023.1-ubuntu_jammy - - 2023.2-ubuntu_jammy + - master-ubuntu_jammy + - 2023.1-ubuntu_jammy + - 2023.2-ubuntu_jammy # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: @@ -47,6 +47,8 @@ jobs: # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Dynamically set MY_DATE environment variable + run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - name: Build and push Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: @@ -55,6 +57,7 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} labels: ${{ steps.meta.outputs.labels }} build-args: | VERSION=${{ github.event.inputs.imageTag }} diff --git a/.github/workflows/release-octavia-ovn.yml b/.github/workflows/release-octavia-ovn.yml index 2551d4b0..d88b22f6 100644 --- a/.github/workflows/release-octavia-ovn.yml +++ b/.github/workflows/release-octavia-ovn.yml @@ -11,14 +11,15 @@ on: default: 'master-ubuntu_jammy' type: choice options: - - master-ubuntu_jammy + - master-ubuntu_jammy pluginTag: description: 'Set plugin version' required: true default: '5.0.0' type: choice options: - - '5.0.0' + - '5.0.0' + # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: REGISTRY: ghcr.io @@ -51,6 +52,8 @@ jobs: # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + - name: Dynamically set MY_DATE environment variable + run: echo "MY_DATE=$(date +%s)" >> $GITHUB_ENV - name: Build and push Docker image uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 with: @@ -59,6 +62,7 @@ jobs: push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:${{ github.event.inputs.imageTag }}-${{ env.MY_DATE }} labels: ${{ steps.meta.outputs.labels }} build-args: | VERSION=${{ github.event.inputs.imageTag }} diff --git a/.github/workflows/smoke-horizon-rxt.yml b/.github/workflows/smoke-horizon-rxt.yml new file mode 100644 index 00000000..5c96b507 --- /dev/null +++ b/.github/workflows/smoke-horizon-rxt.yml @@ -0,0 +1,42 @@ +# +name: Run build check for the Horizon RXT compatible image + +on: + pull_request: + paths: + - Containerfiles/HorizonRXT-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/HorizonRXT-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/horizon-rxt:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy + PLUGIN_VERSION=master diff --git a/.github/workflows/smoke-keystone-rxt.yml b/.github/workflows/smoke-keystone-rxt.yml new file mode 100644 index 00000000..93bbf176 --- /dev/null +++ b/.github/workflows/smoke-keystone-rxt.yml @@ -0,0 +1,41 @@ +# +name: Run build check for the Keystone RXT compatible image + +on: + pull_request: + paths: + - Containerfiles/KeystoneRXT-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/KeystoneRXT-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/keystone-rxt:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy diff --git a/.github/workflows/smoke-nova-uefi.yml b/.github/workflows/smoke-nova-uefi.yml new file mode 100644 index 00000000..9eb52c81 --- /dev/null +++ b/.github/workflows/smoke-nova-uefi.yml @@ -0,0 +1,41 @@ +# +name: Run build check for the Nova EFI compatible image + +on: + pull_request: + paths: + - Containerfiles/NovaEFI-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/NovaEFI-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/nova-efi:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy diff --git a/.github/workflows/smoke-octavia-ovn.yml b/.github/workflows/smoke-octavia-ovn.yml new file mode 100644 index 00000000..cd4c33b2 --- /dev/null +++ b/.github/workflows/smoke-octavia-ovn.yml @@ -0,0 +1,42 @@ +# +name: Run build check for the Octavia OVN compatible image + +on: + pull_request: + paths: + - Containerfiles/OctaviaOVN-Containerfile + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push-image: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: read + steps: + - name: Checkout repository + uses: actions/checkout@v4 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + file: Containerfiles/OctaviaOVN-Containerfile + push: false + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/octavia-ovn:master-ubuntu_jammy + build-args: | + VERSION=master-ubuntu_jammy + PLUGIN_VERSION=5.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..132747f4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + - id: check-byte-order-marker + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + files: .*\.(yaml|yml)$ + args: + - "--allow-multiple-documents" + - "--unsafe" + - id: debug-statements + - repo: https://github.com/psf/black + rev: 24.1.1 + hooks: + - id: black + - repo: https://github.com/ansible-community/ansible-lint + rev: v6.22.2 + hooks: + - id: ansible-lint + additional_dependencies: + - ansible + - yamllint + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 3.0.0 + hooks: + - id: shellcheck diff --git a/Containerfiles/HorizonRXT-Containerfile b/Containerfiles/HorizonRXT-Containerfile index 3fc30a77..696b2c97 100644 --- a/Containerfiles/HorizonRXT-Containerfile +++ b/Containerfiles/HorizonRXT-Containerfile @@ -10,5 +10,6 @@ RUN export ORIG_PLUGIN_VERSION="${PLUGIN_VERSION}"; \ git+https://opendev.org/openstack/heat-dashboard@${PLUGIN_VERSION}#egg=heat_dashboard \ git+https://opendev.org/openstack/octavia-dashboard@${PLUGIN_VERSION}#egg=octavia_dashboard RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + FROM openstackhelm/horizon:${VERSION} COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/KeystoneRXT-Containerfile b/Containerfiles/KeystoneRXT-Containerfile index a0d35ac5..43da64bf 100644 --- a/Containerfiles/KeystoneRXT-Containerfile +++ b/Containerfiles/KeystoneRXT-Containerfile @@ -1,3 +1,8 @@ ARG VERSION=master-ubuntu_jammy -FROM openstackhelm/keystone:$VERSION +FROM openstackhelm/keystone:${VERSION} as build +RUN /var/lib/openstack/bin/pip install --upgrade --force-reinstall pip RUN /var/lib/openstack/bin/pip install keystone-rxt +RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + +FROM openstackhelm/keystone:${VERSION} +COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/Containerfiles/NovaEFI-Containerfile b/Containerfiles/NovaEFI-Containerfile index e8db4abb..c88c2e5b 100644 --- a/Containerfiles/NovaEFI-Containerfile +++ b/Containerfiles/NovaEFI-Containerfile @@ -3,10 +3,10 @@ FROM openstackhelm/nova:$VERSION # Packages for the following features: # - Nova: EFI # - Nova: iSCSI +# Py Packages for the following features: +# - Nova: Libosinfo RUN apt update && apt install -y ovmf open-iscsi multipath-tools libgirepository-1.0-1 libgirepository1.0-dev \ libcairo2-dev python3-dev gcc libosinfo-bin gir1.2-libosinfo-1.0; \ rm -rf /var/cache/apt/archives /var/lib/apt/lists; \ - apt clean -# Packages for the following features: -# - Nova: Libosinfo -RUN /var/lib/openstack/bin/pip install pygobject + apt clean; /var/lib/openstack/bin/pip install pygobject; \ + find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete diff --git a/Containerfiles/OctaviaOVN-Containerfile b/Containerfiles/OctaviaOVN-Containerfile index 3ce7a8f8..ff93aa1c 100644 --- a/Containerfiles/OctaviaOVN-Containerfile +++ b/Containerfiles/OctaviaOVN-Containerfile @@ -1,4 +1,9 @@ ARG VERSION=master-ubuntu_jammy -FROM docker.io/loci/octavia:${VERSION} +FROM docker.io/loci/octavia:${VERSION} as build ARG PLUGIN_VERSION=5.0.0 -RUN /var/lib/openstack/bin/pip install ovn-octavia-provider===${PLUGIN_VERSION} +RUN /var/lib/openstack/bin/pip install --index-url https://pypi.python.org/simple --upgrade pip +RUN /var/lib/openstack/bin/pip install --index-url https://pypi.python.org/simple ovn-octavia-provider==${PLUGIN_VERSION} +RUN find /var/lib/openstack -regex '^.*\(__pycache__\|\.py[co]\)$' -delete + +FROM docker.io/loci/octavia:${VERSION} +COPY --from=build /var/lib/openstack/. /var/lib/openstack/ diff --git a/ansible/inventory/openstack-enterprise/group_vars/k8s_cluster/addons.yml b/ansible/inventory/openstack-enterprise/group_vars/k8s_cluster/addons.yml index 37c96df5..c860ab7a 100644 --- a/ansible/inventory/openstack-enterprise/group_vars/k8s_cluster/addons.yml +++ b/ansible/inventory/openstack-enterprise/group_vars/k8s_cluster/addons.yml @@ -176,7 +176,7 @@ metallb_speaker_enabled: "{{ metallb_enabled }}" metallb_protocol: "layer2" # metallb_port: "7472" # metallb_memberlist_port: "7946" - metallb_config: +metallb_config: speaker: nodeselector: kubernetes.io/os: "linux" diff --git a/requirements.yml b/requirements.yml new file mode 120000 index 00000000..59b55f9b --- /dev/null +++ b/requirements.yml @@ -0,0 +1 @@ +ansible-collection-requirements.yml \ No newline at end of file