Partial merge

canonical · Oct 28, 2023 · 3e80172 · 3e80172
1 parent 6dcc0ab
commit 3e80172
Show file tree

Hide file tree

Showing 5 changed files with 306 additions and 145 deletions.
diff --git a/.github/renovate.json5 b/.github/renovate.json5
@@ -4,88 +4,9 @@
   "reviewers": ["dragomirp", "marceloneppel", "taurus-forever"],
   "packageRules": [
     // Later rules override earlier rules
-    {
-      "matchManagers": ["poetry", "pip_requirements", "regex"],
-      "matchDatasources": ["pypi"],
-      "excludePackageNames": ["juju", "libjuju 2", "libjuju 3"],
-      "groupName": "Python dependencies"
-    }, {
-      "matchManagers": ["github-actions"],
-      "groupName": "GitHub actions"
-    }, {
-      "matchDepNames": ["Juju 3"],
-      "matchPackageNames": ["juju/juju"],
-      "allowedVersions": "<3.2.0",
-      "extractVersion": "^juju-(?<version>.*)$",
-      "groupName": "Juju 3"
-    }, {
-      "matchDepNames": ["Juju 2"],
-      "matchPackageNames": ["juju/juju"],
-      "allowedVersions": "<3.0.0",
-      "extractVersion": "^juju-(?<version>.*)$",
-      "groupName": "Juju 2"
-    }, {
-      "matchDepNames": ["libjuju 3"],
-      "matchPackageNames": ["juju"],
-      "matchDatasources": ["pypi"],
-      "allowedVersions": "<4.0.0",
-      "groupName": "Juju 3"
-    }, {
-      "matchDepNames": ["libjuju 2"],
-      "matchPackageNames": ["juju"],
-      "matchDatasources": ["pypi"],
-      "allowedVersions": "<3.0.0",
-      "groupName": "Juju 2"
-    }, {
-      # Doesn't build on focal, pinned in charm libs
-      "matchPackageNames": ["pydantic"],
-      "allowedVersions": "<2.0.0"
-    }
   ],
   "regexManagers": [
     {
-      "fileMatch": ["^(workflow-templates|\\.github/workflows)/[^/]+\\.ya?ml$"],
-      "matchStrings": ["(- | agent-versions: )\"(?<currentValue>.*?)\" +# renovate: latest juju 2"],
-      "depNameTemplate": "Juju 2",
-      "packageNameTemplate": "juju/juju",
-      "datasourceTemplate": "github-releases",
-      "versioningTemplate": "loose",
-      "extractVersionTemplate": "Juju release"
-    }, {
-      "fileMatch": ["^(workflow-templates|\\.github/workflows)/[^/]+\\.ya?ml$"],
-      "matchStrings": ["(- | agent-versions: )\"(?<currentValue>.*?)\" +# renovate: latest juju 3"],
-      "depNameTemplate": "Juju 3",
-      "packageNameTemplate": "juju/juju",
-      "datasourceTemplate": "github-releases",
-      "versioningTemplate": "loose",
-      "extractVersionTemplate": "Juju release"
-    }, {
-      "fileMatch": [
-        "(^|/)pyproject\\.toml$",
-        "(^|/)tox\\.ini$"
-      ],
-      "matchStrings": [
-        "juju = \".* \\|\\| (?<currentValue>.*?)\" *# renovate libjuju",
-        "LIBJUJU=\"(?<currentValue>.*?)\" *# libjuju3"
-      ],
-      "depNameTemplate": "libjuju 3",
-      "packageNameTemplate": "juju",
-      "datasourceTemplate": "pypi",
-      "versioningTemplate": "loose",
-    }, {
-      "fileMatch": [
-        "(^|/)pyproject\\.toml$",
-        "(^|/)tox\\.ini$"
-      ],
-      "matchStrings": [
-        "juju = \"(?<currentValue>.*?) \\|\\| .*\" *# renovate libjuju",
-        "LIBJUJU=\"(?<currentValue>.*?)\" *# libjuju2"
-      ],
-      "depNameTemplate": "libjuju 2",
-      "packageNameTemplate": "juju",
-      "datasourceTemplate": "pypi",
-      "versioningTemplate": "loose",
-    }, {
       "fileMatch": ["(^|/)([\\w-]*)charmcraft\\.ya?ml$"],
       "matchStrings": ["- (?<depName>.*?)(?:\\[.*?\\])?==(?<currentValue>.*?) +# renovate"],
       "datasourceTemplate": "pypi",

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -46,69 +46,39 @@ jobs:
     permissions:
       actions: write  # Needed to manage GitHub Actions cache
 
-  gh-hosted-collect-integration-tests:
-    name: (GH hosted) Collect integration test groups
-    needs:
-      - lint
-      - unit-test
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v3
-      - name: Install tox & poetry
-        run: |
-          pipx install tox
-          pipx install poetry
-      - name: Select test stability level
-        id: select-test-stability
-        run: |
-          if [[ "${{ github.event_name }}" == "schedule" ]]
-          then
-            echo Running unstable and stable tests
-            echo "mark_expression=" >> "$GITHUB_OUTPUT"
-          else
-            echo Skipping unstable tests
-            echo "mark_expression=not unstable" >> "$GITHUB_OUTPUT"
-          fi
-      - name: Collect test groups
-        id: collect-groups
-        run: tox run -e integration -- tests/integration -m '${{ steps.select-test-stability.outputs.mark_expression }}' --collect-groups
-    outputs:
-      groups: ${{ steps.collect-groups.outputs.groups }}
-
   gh-hosted-integration-test:
     strategy:
       fail-fast: false
       matrix:
-        groups: ${{ fromJSON(needs.gh-hosted-collect-integration-tests.outputs.groups) }}
+        tox-environments:
+          - backup-integration
+          - charm-integration
+          - db-relation-integration
+          - ha-replication-integration
+          - ha-self-healing-integration
+          - ha-restore-cluster-integration
+          - password-rotation-integration
+          - plugins-integration
+          - tls-integration
+          - upgrade-integration
+          - upgrade-from-stable-integration
+          - database-relation-integration
+          - db-admin-relation-integration
         juju-snap-channel:  ["2.9/stable", "3.1/stable"]
+        free-disk-space:
+          - false
         include:
           - juju-snap-channel: "3.1/stable"
             agent-version: "3.1.6"
             libjuju-version:  "3.2.2"
           - juju-snap-channel: "2.9/stable"
             agent-version: "2.9.45"
-            libjuju-version:  "2.9.44.1"
-        exclude:
-          # Disabling HA tests, as long as we want to have a limited pipeline on Juju3
-          - juju-snap-channel: "3.1/stable"
-            groups:
-              job_name: "high_availability/test_replication.py | group 1"
-          - juju-snap-channel: "3.1/stable"
-            groups:
-              job_name: "high_availability/test_self_healing.py | group 1"
-          - juju-snap-channel: "3.1/stable"
-            groups:
-              job_name: "high_availability/test_upgrade.py | group 1"
-          - juju-snap-channel: "3.1/stable"
-            groups:
-              job_name: "high_availability/test_upgrade_from_stable.py | group 1"
-    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.groups.job_name }}
+            libjuju-version:  "2.9.45.0"
+    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.tox-environment }}
     needs:
       - lint
       - unit-test
       - build
-      - gh-hosted-collect-integration-tests
     runs-on: ubuntu-latest
     timeout-minutes: 120
     steps:
@@ -118,6 +88,14 @@ jobs:
         run: |
           pipx install tox
           pipx install poetry
+      - name: Free disk space
+        if: ${{ matrix.free-disk-space == true }}
+        run: |
+          sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+          sudo rm -rf \
+            /usr/share/dotnet /usr/local/lib/android /opt/ghc \
+            /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup \
+            /usr/lib/jvm || true
       - name: Setup operator environment
         uses: charmed-kubernetes/actions-operator@main
         with:
@@ -126,7 +104,7 @@ jobs:
           bootstrap-options: "--agent-version ${{ matrix.agent-version }}"
           juju-channel: ${{ matrix.juju-snap-channel }}
       - name: Update python-libjuju version
-        if: ${{ matrix.libjuju-version == '2.9.44.1' }}
+        if: ${{ matrix.libjuju-version == '2.9.45.0' }}
         run: poetry add --lock --group integration juju@'${{ matrix.libjuju-version }}'
       - name: Download packed charm(s)
         uses: actions/download-artifact@v3
@@ -144,7 +122,7 @@ jobs:
             echo "mark_expression=and not unstable" >> "$GITHUB_OUTPUT"
           fi
       - name: Run integration tests
-        run: tox run -e integration -- "${{ matrix.groups.path_to_test_file }}" --group="${{ matrix.groups.group_number }}" -m '${{ steps.select-test-stability.outputs.mark_expression }}'
+        run: tox run -e ${{ matrix.tox-environments }} -- -m 'test ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models
         env:
           LIBJUJU_VERSION_SPECIFIER: ${{ matrix.libjuju-version }}
           SECRETS_FROM_GITHUB: |

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -69,6 +69,8 @@ pytest-operator = "^0.29.0"
 pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.0.0", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
 pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.0.0", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
 juju = "^3.2.2"
+boto3 = "^1.28.70"
+tenacity = "^8.2.3"
 landscape-api-py3 = "^0.9.0"
 mailmanclient = "^3.3.5"
 psycopg2 = {version = "^2.9.9", extras = ["binary"]}