[DPE-3064] reusable workflow

.github/workflows/ci.yaml

@@ -22,7 +22,7 @@ on:
 jobs:
   lint:
     name: Lint
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].2
 
   unit-test:
     name: Unit test charm
@@ -45,83 +45,35 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].2
     permissions:
       actions: write  # Needed to manage GitHub Actions cache
 
-  gh-hosted-integration-test:
+  integration-test:
     strategy:
       fail-fast: false
       matrix:
-        tox-environment:
-          - backup-integration
-          - charm-integration
-          - database-relation-integration
-          - db-relation-integration
-          - db-admin-relation-integration
-          - ha-replication-integration
-          - ha-self-healing-integration
-          - password-rotation-integration
-          - plugins-integration
-          - tls-integration
-          - upgrade-integration
-          - upgrade-from-stable-integration
-        juju-snap-channel:  ["2.9/stable", "3.1/stable"]
-        include:
-          - juju-snap-channel: "3.1/stable"
-            agent-version: "3.1.6"
-            libjuju-version:  "3.2.2"
-            exclude-mark:  "juju2"
-          - juju-snap-channel: "2.9/stable"
-            agent-version: "2.9.45"
-            libjuju-version:  "2.9.45.0"
-            exclude-mark:  "juju3"
-    name: ${{ matrix.juju-snap-channel }} - (GH hosted) ${{ matrix.tox-environment }}
+        juju:
+          - agent: 2.9.45
+            libjuju: ^2
+          - agent: 3.1.6
+    name: Integration test charm | ${{ matrix.juju.agent }}
     needs:
       - lint
       - unit-test
       - build
-    runs-on: ubuntu-latest
-    timeout-minutes: 120
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Install tox & poetry
-        run: |
-          pipx install tox
-          pipx install poetry
-      - name: Setup operator environment
-        uses: charmed-kubernetes/actions-operator@main
-        with:
-          provider: microk8s
-          channel: "1.28-strict/stable"
-          bootstrap-options: "--agent-version ${{ matrix.agent-version }}"
-          juju-channel: ${{ matrix.juju-snap-channel }}
-      - name: Update python-libjuju version
-        if: ${{ matrix.juju-snap-channel == '2.9/stable' }}
-        run: poetry add --lock --group integration juju@'${{ matrix.libjuju-version }}'
-      - name: Download packed charm(s)
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ needs.build.outputs.artifact-name }}
-      - name: Select test stability level
-        id: select-test-stability
-        run: |
-          if [[ "${{ github.event_name }}" == "schedule" ]]
-          then
-            echo Running unstable and stable tests
-            echo "mark_expression=" >> "$GITHUB_OUTPUT"
-          else
-            echo Skipping unstable tests
-            echo "mark_expression=and not unstable" >> "$GITHUB_OUTPUT"
-          fi
-      - name: Run integration tests
-        run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models
-        env:
-          SECRETS_FROM_GITHUB: |
-            {
-              "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
-              "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
-              "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
-              "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
-            }
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected]
+    with:
+      artifact-name: ${{ needs.build.outputs.artifact-name }}
+      cloud: microk8s
+      microk8s-snap-channel: 1.28-strict/stable
+      juju-agent-version: ${{ matrix.juju.agent }}
+      libjuju-version-constraint: ${{ matrix.juju.libjuju }}
+    secrets:
+      integration-test: |
+        {
+          "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}",
+          "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}",
+          "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}",
+          "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}",
+        }

.github/workflows/release.yaml

@@ -41,14 +41,14 @@ jobs:
 
   build:
     name: Build charm
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].2
 
   release:
     name: Release charm
     needs:
       - ci-tests
       - build
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].2
     with:
       channel: 14/edge
       artifact-name: ${{ needs.build.outputs.artifact-name }}

.github/workflows/sync_issue_to_jira.yaml

@@ -9,7 +9,7 @@ on:
 jobs:
   sync:
     name: Sync GitHub issue to Jira
-    uses: canonical/data-platform-workflows/.github/workflows/[email protected].1
+    uses: canonical/data-platform-workflows/.github/workflows/[email protected].2
     with:
       jira-base-url: https://warthogs.atlassian.net
       jira-project-key: DPE

pyproject.toml

@@ -70,10 +70,10 @@ optional = true
 [tool.poetry.group.integration.dependencies]
 lightkube = "^0.15.0"
 pytest = "^7.4.3"
-pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.1", subdirectory = "python/pytest_plugins/github_secrets"}
+pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.2", subdirectory = "python/pytest_plugins/github_secrets"}
 pytest-operator = "^0.29.0"
-pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.1", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
-pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.1", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
+pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.2", subdirectory = "python/pytest_plugins/pytest_operator_cache"}
+pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.3.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"}
 juju = "^3.2.2"
 psycopg2 = {version = "^2.9.9", extras = ["binary"]}
 boto3 = "^1.28.70"
@@ -96,7 +96,7 @@ show_missing = true
 minversion = "6.0"
 log_cli_level = "INFO"
 asyncio_mode = "auto"
-markers = ["unstable"]
+markers = ["unstable", "juju2", "juju3", "juju_secrets"]
 
 # Formatting tools configuration
 [tool.black]

tests/integration/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/ha_tests/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2023 Canonical Ltd.
+# See LICENSE file for licensing details.

tests/integration/ha_tests/conftest.py

@@ -6,7 +6,8 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_delay, wait_fixed
 
-from tests.integration.ha_tests.helpers import (
+from ..helpers import app_name
+from .helpers import (
     APPLICATION_NAME,
     change_patroni_setting,
     change_wal_settings,
@@ -17,7 +18,6 @@
     modify_pebble_restart_delay,
     remove_instance_isolation,
 )
-from tests.integration.helpers import app_name
 
 
 @pytest.fixture()

tests/integration/ha_tests/helpers.py

@@ -27,7 +27,7 @@
     wait_fixed,
 )
 
-from tests.integration.helpers import (
+from ..helpers import (
     APPLICATION_NAME,
     app_name,
     db_connect,

tests/integration/ha_tests/test_replication.py

@@ -7,13 +7,7 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_delay, wait_fixed
 
-from tests.integration.ha_tests.helpers import (
-    are_writes_increasing,
-    check_writes,
-    is_cluster_updated,
-    start_continuous_writes,
-)
-from tests.integration.helpers import (
+from ..helpers import (
     APPLICATION_NAME,
     CHARM_SERIES,
     app_name,
@@ -24,8 +18,15 @@
     get_unit_address,
     scale_application,
 )
+from .helpers import (
+    are_writes_increasing,
+    check_writes,
+    is_cluster_updated,
+    start_continuous_writes,
+)
 
 
+@pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
     """Build and deploy three unit of PostgreSQL."""
@@ -51,6 +52,7 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
             await ops_test.model.wait_for_idle(status="active", timeout=1000)
 
 
+@pytest.mark.group(1)
 async def test_reelection(ops_test: OpsTest, continuous_writes, primary_start_timeout) -> None:
     """Kill primary unit, check reelection."""
     app = await app_name(ops_test)
@@ -82,6 +84,7 @@ async def test_reelection(ops_test: OpsTest, continuous_writes, primary_start_ti
     await is_cluster_updated(ops_test, primary_name)
 
 
+@pytest.mark.group(1)
 async def test_consistency(ops_test: OpsTest, continuous_writes) -> None:
     """Write to primary, read data from secondaries (check consistency)."""
     # Locate primary unit.
@@ -99,6 +102,7 @@ async def test_consistency(ops_test: OpsTest, continuous_writes) -> None:
     await check_writes(ops_test)
 
 
+@pytest.mark.group(1)
 async def test_no_data_replicated_between_clusters(ops_test: OpsTest, continuous_writes) -> None:
     """Check that writes in one cluster are not replicated to another cluster."""
     # Locate primary unit.

tests/integration/ha_tests/test_self_healing.py

@@ -9,7 +9,19 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_delay, wait_fixed
 
-from tests.integration.ha_tests.helpers import (
+from .. import markers
+from ..helpers import (
+    APPLICATION_NAME,
+    CHARM_SERIES,
+    METADATA,
+    app_name,
+    build_and_deploy,
+    db_connect,
+    get_password,
+    get_unit_address,
+    run_command_on_unit,
+)
+from .helpers import (
     are_all_db_processes_down,
     are_writes_increasing,
     change_patroni_setting,
@@ -29,17 +41,6 @@
     send_signal_to_process,
     start_continuous_writes,
 )
-from tests.integration.helpers import (
-    APPLICATION_NAME,
-    CHARM_SERIES,
-    METADATA,
-    app_name,
-    build_and_deploy,
-    db_connect,
-    get_password,
-    get_unit_address,
-    run_command_on_unit,
-)
 
 logger = logging.getLogger(__name__)
 
@@ -50,6 +51,7 @@
 MEDIAN_ELECTION_TIME = 10
 
 
+@pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
     """Build and deploy three unit of PostgreSQL."""
@@ -75,7 +77,8 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
             await ops_test.model.wait_for_idle(status="active", timeout=1000)
 
 
-@pytest.mark.juju2
+@pytest.mark.group(1)
+@markers.juju2
 @pytest.mark.parametrize("process", DB_PROCESSES)
 async def test_kill_db_process(
     ops_test: OpsTest, process: str, continuous_writes, primary_start_timeout
@@ -106,6 +109,7 @@ async def test_kill_db_process(
     await is_cluster_updated(ops_test, primary_name)
 
 
+@pytest.mark.group(1)
 @pytest.mark.parametrize("process", DB_PROCESSES)
 async def test_freeze_db_process(
     ops_test: OpsTest, process: str, continuous_writes, primary_start_timeout
@@ -147,6 +151,7 @@ async def test_freeze_db_process(
     await is_cluster_updated(ops_test, primary_name)
 
 
+@pytest.mark.group(1)
 @pytest.mark.parametrize("process", DB_PROCESSES)
 async def test_restart_db_process(
     ops_test: OpsTest, process: str, continuous_writes, primary_start_timeout
@@ -177,6 +182,7 @@ async def test_restart_db_process(
     await is_cluster_updated(ops_test, primary_name)
 
 
+@pytest.mark.group(1)
 @pytest.mark.unstable
 @pytest.mark.parametrize("process", DB_PROCESSES)
 @pytest.mark.parametrize("signal", ["SIGTERM", "SIGKILL"])
@@ -243,6 +249,7 @@ async def test_full_cluster_restart(
     await check_writes(ops_test)
 
 
+@pytest.mark.group(1)
 @pytest.mark.ha_self_healing_tests
 async def test_forceful_restart_without_data_and_transaction_logs(
     ops_test: OpsTest,
@@ -326,6 +333,7 @@ async def test_forceful_restart_without_data_and_transaction_logs(
     await is_cluster_updated(ops_test, primary_name)
 
 
+@pytest.mark.group(1)
 async def test_network_cut(
     ops_test: OpsTest, continuous_writes, primary_start_timeout, chaos_mesh
 ) -> None: