merge with sqre latest version

lsst-uk · Aug 12, 2024 · 30b8053 · 30b8053
2 parents c08b618 + 3bb8f0e
commit 30b8053
Show file tree

Hide file tree

Showing 1,720 changed files with 58,381 additions and 21,390 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -9,23 +9,25 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
+        with:
+          go-version: '>=1.22'
 
       - name: Install helm-docs
-        run: go install github.com/norwoodj/helm-docs/cmd/helm-docs@latest
+        run: go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.13.1
         env:
           GOBIN: /usr/local/bin/
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit
-        uses: pre-commit/[email protected].0
+        uses: pre-commit/[email protected].1
 
   test:
     runs-on: ubuntu-latest
@@ -35,47 +37,50 @@ jobs:
       matrix:
         python:
           - "3.11"
+          - "3.12"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - uses: lsst-sqre/run-tox@v1
         with:
           python-version: ${{ matrix.python }}
           tox-envs: "typing,py,coverage-report"
+          tox-requirements: requirements/tox.txt
           cache-key-prefix: test
 
   helm:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - uses: azure/setup-helm@v4
         with:
-          python-version: "3.11"
+          # Used to query GitHub for the latest Helm release.
+          token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Install test dependencies
-        run: make init
+      - name: Install helm-unittest plugin
+        run: helm plugin install https://github.com/helm-unittest/helm-unittest
 
-      - name: Expand modified charts
-        run: expand-charts
+      - name: Run helm unittest
+        run: helm unittest {applications,charts}/*/tests/..
 
-      - name: Set up chart-testing
-        uses: helm/[email protected]
-
-      - name: Run chart-testing (lint)
-        run: ct lint --all --config ct.yaml
+      - uses: lsst-sqre/run-tox@v1
+        with:
+          python-version: "3.12"
+          tox-envs: phalanx-lint-change
+          tox-requirements: requirements/tox.txt
+          cache-key-prefix: helm
 
   # The minikube job always runs, but it quickly does nothing if no files that
-  # would affect minikube were changed.  This unfortunately requires a lot of
+  # would affect minikube were changed. This unfortunately requires a lot of
   # if conditionals on all the steps of the job, but we need the job to run so
   # that we can make it mandatory before merging, which in turn allows us to
-  # use automerge.
+  # use merge queues.
   minikube:
     name: Test deploy
     runs-on: ubuntu-latest
@@ -84,27 +89,38 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Filter paths
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
             minikube:
               - ".github/workflows/ci.yaml"
-              - "applications/*/Chart.yaml"
-              - "applications/*/templates/**"
-              - "applications/*/values.yaml"
-              - "applications/*/values-minikube.yaml"
+              - "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,vault-secrets-operator}/Chart.yaml"
+              - "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,vault-secrets-operator}/templates/**"
+              - "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,vault-secrets-operator}/values.yaml"
+              - "applications/{argocd,gafaelfawr,ingress-nginx,mobu,postgres,vault-secrets-operator}/values-minikube.yaml"
               - "environments/Chart.yaml"
-              - "environments/templates/**"
+              - "environments/templates/applications/infrastructure/*"
               - "environments/values-minikube.yaml"
-              - "installer/**"
+              - "src/phalanx/**"
+
+      - name: Download installer dependencies
+        if: steps.filter.outputs.minikube == 'true'
+        run: |
+          curl -sSL -o /tmp/vault.zip https://releases.hashicorp.com/vault/1.15.4/vault_1.15.4_linux_amd64.zip
+          unzip /tmp/vault.zip
+          sudo mv vault /usr/local/bin/vault
+          sudo chmod +x /usr/local/bin/vault
+          sudo curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/download/v2.8.6/argocd-linux-amd64
+          sudo chmod +x /usr/local/bin/argocd
+          sudo apt-get install socat
 
       - name: Setup Minikube
         if: steps.filter.outputs.minikube == 'true'
-        uses: medyagh/[email protected].14
+        uses: medyagh/[email protected].18
         with:
           kubernetes-version: "v1.27.3"
           cpus: max
@@ -114,24 +130,19 @@ jobs:
         if: steps.filter.outputs.minikube == 'true'
         run: kubectl get nodes
 
-      - name: Download installer dependencies
+      - uses: lsst-sqre/run-tox@v1
         if: steps.filter.outputs.minikube == 'true'
-        run: |
-          curl -sSL -o /tmp/vault.zip https://releases.hashicorp.com/vault/1.14.0/vault_1.14.0_linux_amd64.zip
-          unzip /tmp/vault.zip
-          sudo mv vault /usr/local/bin/vault
-          sudo chmod +x /usr/local/bin/vault
-          sudo curl -sSL -o /usr/local/bin/argocd https://github.com/argoproj/argo-cd/releases/download/v2.7.8/argocd-linux-amd64
-          sudo chmod +x /usr/local/bin/argocd
-          sudo apt-get install socat
-          sudo pip install -r installer/requirements.txt
-
-      - name: Run installer
         timeout-minutes: 15
-        if: steps.filter.outputs.minikube == 'true'
-        run: |
-          cd installer
-          ./install.sh minikube ${{ secrets.MINIKUBE_VAULT_KEY }}
+        with:
+          python-version: "3.12"
+          tox-envs: install
+          tox-requirements: requirements/tox.txt
+          tox-posargs: >-
+            --force-noninteractive
+            --vault-role-id=${{ secrets.MINIKUBE_VAULT_ROLE_ID }}
+            --vault-secret-id=${{ secrets.MINIKUBE_VAULT_SECRET_ID }}
+            minikube
+          cache-key-prefix: test
 
       - name: Get final list of resources
         if: steps.filter.outputs.minikube == 'true'

diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml
@@ -1,6 +1,8 @@
 name: Dependency Update
 
 "on":
+  schedule:
+    - cron: "0 12 * * 1"
   workflow_dispatch: {}
 
 jobs:
@@ -9,14 +11,16 @@ jobs:
     timeout-minutes: 10
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
+      # Omit pre-commit updates for now until neophile looks only at releases
+      # so that it doesn't pick up an old helm-docs release.
       - name: Run neophile
         uses: lsst-sqre/run-neophile@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           mode: pr
-          types: pre-commit
+          types: python
           app-id: ${{ secrets.NEOPHILE_APP_ID }}
           app-secret: ${{ secrets.NEOPHILE_PRIVATE_KEY }}
 

diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml
@@ -14,8 +14,6 @@ name: Docs
       - "renovate/**"
       - "tickets/**"
       - "u/**"
-    tags:
-      - "*"
   workflow_dispatch: {}
 
 jobs:
@@ -24,13 +22,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Print GitHub event name
         run: echo "${{ github.event_name }}"
 
       - name: Filter paths
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |
@@ -39,24 +37,30 @@ jobs:
               - "docs/**"
               - "applications/*/Chart.yaml"
               - "applications/*/values.yaml"
+              - "applications/argocd/values-*.yaml"
               - "applications/gafaelfawr/values-*.yaml"
               - "environments/values-*.yaml"
+              - "src/phalanx/**"
+            docsSpecific:
+              - "docs/**"
 
       - name: Install graphviz
-        if: steps.filter.outputs.docs == 'true'
+        if: steps.filter.outputs.docs == 'true' || github.event_name == 'workflow_dispatch'
         run: sudo apt-get install graphviz
 
       - name: Build docs
-        if: steps.filter.outputs.docs == 'true'
+        if: steps.filter.outputs.docs == 'true' || github.event_name == 'workflow_dispatch'
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           tox-envs: docs
+          tox-requirements: requirements/tox.txt
+          cache-key-prefix: docs
 
-      # Only attempt documentation uploads for tagged releases and pull
-      # requests from ticket branches in the same repository. This avoids
-      # version clutter in the docs and failures when a PR doesn't have access
-      # to secrets.
+      # Upload docs:
+      # - on pushes to main if *any* documentation content might have changed
+      # - on workflow dispatches if any documentation content might have changed
+      # - on pushes to tickets/ branches if docs/ directory content changed
       - name: Upload to LSST the Docs
         uses: lsst-sqre/ltd-upload@v1
         with:
@@ -65,7 +69,6 @@ jobs:
           username: ${{ secrets.LTD_USERNAME }}
           password: ${{ secrets.LTD_PASSWORD }}
         if: >-
-          steps.filter.outputs.docs == 'true'
-          && github.event_name != 'merge_group'
-          && (github.event_name != 'pull_request'
-              || startsWith(github.head_ref, 'tickets/'))
+          (github.event_name == 'push' && github.ref_name == 'main' && steps.filter.outputs.docs == 'true')
+          || (github.event_name == 'workflow_dispatch')
+          || (github.event_name == 'pull_request' && startsWith(github.head_ref, 'tickets/') && steps.filter.outputs.docsSpecific == 'true')
diff --git a/.github/workflows/linkcheck.yaml b/.github/workflows/linkcheck.yaml
@@ -1,6 +1,6 @@
 # This is a separate documentation build just to check links.  We don't check
 # links as part of the normal documentation build since, unlike Sphinx errors
-# and warnings, we don't want broken links to block a merge.  (Sometimes they
+# and warnings, we don't want broken links to block a merge. (Sometimes they
 # will be fixed by the same merge, sometimes they're temporary rate limit
 # issues.)
 #
@@ -40,13 +40,15 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install graphviz
         run: sudo apt-get install graphviz
 
       - name: Check links
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           tox-envs: docs-linkcheck
+          tox-requirements: requirements/tox.txt
+          cache-key-prefix: docs-linkcheck
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,3 @@
-/installer/secrets/
-/installer/docker-creds
 /applications/*/charts/*.tgz
 /applications-expanded/
 **/Chart.lock

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,50 +1,59 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: check-merge-conflict
       - id: check-toml
       - id: trailing-whitespace
 
   - repo: https://github.com/adrienverge/yamllint
-    rev: v1.32.0
+    rev: v1.35.1
     hooks:
       - id: yamllint
         args:
           - -c=.yamllint.yml
 
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.23.3
+    rev: 0.29.1
     hooks:
       - id: check-jsonschema
-        files: "^applications/.*/secrets(-[^./-]+)?\\.yaml"
-        args: ["--schemafile", "docs/extras/schemas/secrets.json"]
+        files: ^applications/.*/secrets(-[^./-]+)?\.yaml
+        args: [--schemafile, docs/extras/schemas/secrets.json]
+      - id: check-jsonschema
+        files: ^environments/values(-[^./-]+)?\.yaml
+        args: [--schemafile, docs/extras/schemas/environment.json]
       - id: check-metaschema
-        files: "^docs/extras/schemas/.*\\.json"
+        files: ^docs/extras/schemas/.*\.json
 
   - repo: https://github.com/norwoodj/helm-docs
-    rev: v1.11.1
+    rev: v1.14.2
     hooks:
       - id: helm-docs
         args:
-          - --chart-search-root=.
-          # The `./` makes it relative to the chart-search-root set above
-          - --template-files=./helm-docs.md.gotmpl
+          - --chart-search-root=applications
+          # The `../` makes it relative to the chart-search-root set above
+          - --template-files=../helm-docs.md.gotmpl
           - --document-dependency-values=true
+      - id: helm-docs
+        args:
+          - --chart-search-root=environments
+          # The `../` makes it relative to the chart-search-root set above
+          - --template-files=../helm-docs.md.gotmpl
+      - id: helm-docs
+        args:
+          - --chart-search-root=charts
+          # The `../` makes it relative to the chart-search-root set above
+          - --template-files=../helm-docs.md.gotmpl
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.280
+    rev: v0.5.5
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-
-  - repo: https://github.com/psf/black
-    rev: 23.7.0
-    hooks:
-      - id: black
+      - id: ruff-format
 
   - repo: https://github.com/adamchainz/blacken-docs
-    rev: 1.15.0
+    rev: 1.18.0
     hooks:
       - id: blacken-docs
         additional_dependencies: [black==23.7.0]