diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb4d4849ca..43805ce9ce 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,7 +22,7 @@ on: jobs: lint: name: Lint - uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v5.1.2 + uses: canonical/data-platform-workflows/.github/workflows/lint.yaml@v6.1.1 unit-test: name: Unit test charm @@ -45,7 +45,7 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v5.1.2 + uses: canonical/data-platform-workflows/.github/workflows/build_charms_with_cache.yaml@v6.1.1 permissions: actions: write # Needed to manage GitHub Actions cache @@ -115,14 +115,8 @@ jobs: echo Skipping unstable tests echo "mark_expression=and not unstable" >> "$GITHUB_OUTPUT" fi - - name: Select test secret usage - id: select-test-secrets - if: ${{ github.event.pull_request.head.repo.full_name != 'canonical/postgresql-k8s-operator' }} - run: | - echo Skipping tests using secrets - echo "mark_secrets=and not uses_secrets" >> "$GITHUB_OUTPUT" - name: Run integration tests - run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-secrets.outputs.mark_secrets }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models + run: tox run -e ${{ matrix.tox-environment }} -- -m 'not ${{ matrix.exclude-mark }} ${{ steps.select-test-stability.outputs.mark_expression }}' --keep-models env: SECRETS_FROM_GITHUB: | { diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 09006b1bd4..3497bea2a3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -30,7 +30,7 @@ jobs: timeout-minutes: 60 steps: - name: Checkout - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Release charm libraries @@ -41,14 +41,14 @@ jobs: build: name: Build charm - uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v5.1.2 + uses: canonical/data-platform-workflows/.github/workflows/build_charm_without_cache.yaml@v6.1.1 release: name: Release charm needs: - ci-tests - build - uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v5.1.2 + uses: canonical/data-platform-workflows/.github/workflows/release_charm.yaml@v6.1.1 with: channel: 14/edge artifact-name: ${{ needs.build.outputs.artifact-name }} diff --git a/.github/workflows/sync_issue_to_jira.yaml b/.github/workflows/sync_issue_to_jira.yaml index 21624e4ece..d500e30fcd 100644 --- a/.github/workflows/sync_issue_to_jira.yaml +++ b/.github/workflows/sync_issue_to_jira.yaml @@ -9,7 +9,7 @@ on: jobs: sync: name: Sync GitHub issue to Jira - uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v5.1.2 + uses: canonical/data-platform-workflows/.github/workflows/sync_issue_to_jira.yaml@v6.1.1 with: jira-base-url: https://warthogs.atlassian.net jira-project-key: DPE diff --git a/poetry.lock b/poetry.lock index c79c6a17ce..8254197ffc 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1462,8 +1462,8 @@ develop = false [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v5.1.2" -resolved_reference = "3cc668dc10fa7316da9600c296ca7640d7d83222" +reference = "v6.1.1" +resolved_reference = "c9bf1f5fb128800ed4f0a317509884f75c63902b" subdirectory = "python/pytest_plugins/github_secrets" [[package]] @@ -1517,8 +1517,8 @@ pyyaml = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v5.1.2" -resolved_reference = "3cc668dc10fa7316da9600c296ca7640d7d83222" +reference = "v6.1.1" +resolved_reference = "c9bf1f5fb128800ed4f0a317509884f75c63902b" subdirectory = "python/pytest_plugins/pytest_operator_cache" [[package]] @@ -1536,8 +1536,8 @@ pytest = "*" [package.source] type = "git" url = "https://github.com/canonical/data-platform-workflows" -reference = "v5.1.2" -resolved_reference = "3cc668dc10fa7316da9600c296ca7640d7d83222" +reference = "v6.1.1" +resolved_reference = "c9bf1f5fb128800ed4f0a317509884f75c63902b" subdirectory = "python/pytest_plugins/pytest_operator_groups" [[package]] @@ -2079,4 +2079,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "da2c83c7697cc101594c002dd42b78e06777ac9605aa40798ecb276cf561e27a" +content-hash = "a92bc7ee5adca12673a90b59e900ee33d2300d78c3e7efcee56c3fc867735822" diff --git a/pyproject.toml b/pyproject.toml index ed3e7f7cd9..cb6df45c62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,10 +69,10 @@ optional = true [tool.poetry.group.integration.dependencies] lightkube = "^0.14.0" pytest = "^7.4.3" -pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.1.2", subdirectory = "python/pytest_plugins/github_secrets"} +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.1.1", subdirectory = "python/pytest_plugins/github_secrets"} pytest-operator = "^0.29.0" -pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.1.2", subdirectory = "python/pytest_plugins/pytest_operator_cache"} -pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.1.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"} +pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.1.1", subdirectory = "python/pytest_plugins/pytest_operator_cache"} +pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v6.1.1", subdirectory = "python/pytest_plugins/pytest_operator_groups"} juju = "^3.2.2" psycopg2 = {version = "^2.9.9", extras = ["binary"]} boto3 = "^1.28.70" diff --git a/tests/integration/ha_tests/helpers.py b/tests/integration/ha_tests/helpers.py index 427239b2bd..e1786ca1f9 100644 --- a/tests/integration/ha_tests/helpers.py +++ b/tests/integration/ha_tests/helpers.py @@ -7,13 +7,11 @@ import tarfile import tempfile from datetime import datetime -from pathlib import Path from typing import Dict, Optional, Set, Tuple import kubernetes as kubernetes import psycopg2 import requests -import yaml from kubernetes import config from kubernetes.client.api import core_v1_api from kubernetes.stream import stream @@ -30,6 +28,7 @@ ) from tests.integration.helpers import ( + APPLICATION_NAME, app_name, db_connect, get_password, @@ -37,8 +36,6 @@ get_unit_address, ) -APPLICATION_NAME = "postgresql-test-app" -METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) PORT = 5432 diff --git a/tests/integration/ha_tests/test_replication.py b/tests/integration/ha_tests/test_replication.py index e9496000d6..506ed00dba 100644 --- a/tests/integration/ha_tests/test_replication.py +++ b/tests/integration/ha_tests/test_replication.py @@ -8,13 +8,13 @@ from tenacity import Retrying, stop_after_delay, wait_fixed from tests.integration.ha_tests.helpers import ( - APPLICATION_NAME, are_writes_increasing, check_writes, is_cluster_updated, start_continuous_writes, ) from tests.integration.helpers import ( + APPLICATION_NAME, CHARM_SERIES, app_name, build_and_deploy, diff --git a/tests/integration/ha_tests/test_self_healing.py b/tests/integration/ha_tests/test_self_healing.py index 6913672087..0ac5fa74c5 100644 --- a/tests/integration/ha_tests/test_self_healing.py +++ b/tests/integration/ha_tests/test_self_healing.py @@ -10,8 +10,6 @@ from tenacity import Retrying, stop_after_delay, wait_fixed from tests.integration.ha_tests.helpers import ( - APPLICATION_NAME, - METADATA, are_all_db_processes_down, are_writes_increasing, change_patroni_setting, @@ -32,7 +30,9 @@ start_continuous_writes, ) from tests.integration.helpers import ( + APPLICATION_NAME, CHARM_SERIES, + METADATA, app_name, build_and_deploy, db_connect, diff --git a/tests/integration/ha_tests/test_upgrade.py b/tests/integration/ha_tests/test_upgrade.py index 1d7a81b3b8..52ca535d7a 100644 --- a/tests/integration/ha_tests/test_upgrade.py +++ b/tests/integration/ha_tests/test_upgrade.py @@ -16,12 +16,12 @@ from tenacity import Retrying, stop_after_attempt, wait_fixed from tests.integration.ha_tests.helpers import ( - APPLICATION_NAME, are_writes_increasing, check_writes, start_continuous_writes, ) from tests.integration.helpers import ( + APPLICATION_NAME, DATABASE_APP_NAME, METADATA, count_switchovers, diff --git a/tests/integration/ha_tests/test_upgrade_from_stable.py b/tests/integration/ha_tests/test_upgrade_from_stable.py index 34068a0f33..7693ba4dd1 100644 --- a/tests/integration/ha_tests/test_upgrade_from_stable.py +++ b/tests/integration/ha_tests/test_upgrade_from_stable.py @@ -11,12 +11,12 @@ from tenacity import Retrying, stop_after_attempt, wait_fixed from tests.integration.ha_tests.helpers import ( - APPLICATION_NAME, are_writes_increasing, check_writes, start_continuous_writes, ) from tests.integration.helpers import ( + APPLICATION_NAME, DATABASE_APP_NAME, METADATA, count_switchovers, diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 0cc71cdac6..e0b1ced0e0 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -32,6 +32,7 @@ CHARM_SERIES = "jammy" METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) DATABASE_APP_NAME = METADATA["name"] +APPLICATION_NAME = "postgresql-test-app" charm = None diff --git a/tests/integration/new_relations/test_new_relations.py b/tests/integration/new_relations/test_new_relations.py index b7b6224190..db3f072efa 100644 --- a/tests/integration/new_relations/test_new_relations.py +++ b/tests/integration/new_relations/test_new_relations.py @@ -569,3 +569,39 @@ async def test_invalid_extra_user_roles(ops_test: OpsTest): raise_on_blocked=False, timeout=1000, ) + + +async def test_indico_datatabase(ops_test: OpsTest) -> None: + """Tests deploying and relating to the Indico charm.""" + async with ops_test.fast_forward(fast_interval="30s"): + await ops_test.model.deploy( + "indico", + channel="stable", + application_name="indico", + num_units=1, + ) + await ops_test.model.deploy("redis-k8s", channel="stable", application_name="redis-broker") + await ops_test.model.deploy("redis-k8s", channel="stable", application_name="redis-cache") + await asyncio.gather( + ops_test.model.relate("redis-broker", "indico"), + ops_test.model.relate("redis-cache", "indico"), + ) + + # Wait for model to stabilise + await ops_test.model.wait_for_idle( + apps=["indico"], + status="waiting", + timeout=1000, + ) + + # Verify that the charm doesn't block when the extensions are enabled. + logger.info("Verifying that the charm doesn't block when the extensions are enabled") + config = {"plugin_pg_trgm_enable": "True", "plugin_unaccent_enable": "True"} + await ops_test.model.applications[DATABASE_APP_NAME].set_config(config) + await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active") + await ops_test.model.relate(DATABASE_APP_NAME, "indico") + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME, "indico"], + status="active", + timeout=2000, + ) diff --git a/tests/integration/test_backups.py b/tests/integration/test_backups.py index 17eb9f8737..73d224c667 100644 --- a/tests/integration/test_backups.py +++ b/tests/integration/test_backups.py @@ -85,7 +85,6 @@ async def test_none() -> None: pass -@pytest.mark.uses_secrets @pytest.mark.abort_on_fail async def test_backup_and_restore(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> None: """Build and deploy two units of PostgreSQL and then test the backup and restore actions.""" @@ -209,8 +208,7 @@ async def test_backup_and_restore(ops_test: OpsTest, cloud_configs: Tuple[Dict, await ops_test.model.remove_application(TLS_CERTIFICATES_APP_NAME, block_until_done=True) -@pytest.mark.uses_secrets -async def test_restore_on_new_cluster(ops_test: OpsTest) -> None: +async def test_restore_on_new_cluster(ops_test: OpsTest, github_secrets) -> None: """Test that is possible to restore a backup to another PostgreSQL cluster.""" database_app_name = f"new-{DATABASE_APP_NAME}" await build_and_deploy(ops_test, 1, database_app_name=database_app_name, wait_for_idle=False) @@ -282,7 +280,6 @@ async def test_restore_on_new_cluster(ops_test: OpsTest) -> None: connection.close() -@pytest.mark.uses_secrets async def test_invalid_config_and_recovery_after_fixing_it( ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict] ) -> None: diff --git a/tests/integration/test_db.py b/tests/integration/test_db.py index a09a63a085..b534d955bf 100644 --- a/tests/integration/test_db.py +++ b/tests/integration/test_db.py @@ -8,11 +8,14 @@ from pytest_operator.plugin import OpsTest from tests.integration.helpers import ( + APPLICATION_NAME, + CHARM_SERIES, DATABASE_APP_NAME, build_and_deploy, check_database_creation, check_database_users_existence, deploy_and_relate_application_with_postgresql, + get_leader_unit, get_primary, wait_for_relation_removed_between, ) @@ -85,148 +88,94 @@ async def test_finos_waltz_db(ops_test: OpsTest) -> None: # Remove the first deployment of Finos Waltz. await ops_test.model.remove_application(FINOS_WALTZ_APP_NAME, block_until_done=True) - # Remove the PostgreSQL application. - await ops_test.model.remove_application(DATABASE_APP_NAME, block_until_done=True) +async def test_extensions_blocking(ops_test: OpsTest) -> None: + await ops_test.model.deploy( + APPLICATION_NAME, + application_name=APPLICATION_NAME, + series=CHARM_SERIES, + channel="edge", + ) + await ops_test.model.deploy( + APPLICATION_NAME, + application_name=f"{APPLICATION_NAME}2", + series=CHARM_SERIES, + channel="edge", + ) -@pytest.mark.skip(reason="Should be ported and moved to the new relation tests") -async def test_indico_db_blocked(ops_test: OpsTest) -> None: - """Tests if deploying and relating to Indico charm will block due to requested extensions.""" - async with ops_test.fast_forward(fast_interval="30s"): - # Build and deploy the PostgreSQL charm (use a custom name until - # https://warthogs.atlassian.net/browse/DPE-2000 is solved). - database_application_name = f"extensions-{DATABASE_APP_NAME}" - await build_and_deploy(ops_test, 1, database_application_name) - - await ops_test.model.deploy( - "indico", - channel="stable", - application_name="indico1", - num_units=APPLICATION_UNITS, - ) - await ops_test.model.deploy( - "indico", - channel="stable", - application_name="indico2", - num_units=APPLICATION_UNITS, - ) - await ops_test.model.deploy("redis-k8s", channel="stable", application_name="redis-broker") - await ops_test.model.deploy("redis-k8s", channel="stable", application_name="redis-cache") - await gather( - ops_test.model.relate("redis-broker", "indico1"), - ops_test.model.relate("redis-cache", "indico1"), - ) - - # Wait for model to stabilise - await ops_test.model.wait_for_idle( - apps=["indico1", "indico2"], - status="waiting", - raise_on_blocked=False, - timeout=1000, - ) - unit = ops_test.model.units.get("indico1/0") - ops_test.model.block_until( - lambda: unit.workload_status_message == "Waiting for database availability", - timeout=1000, - ) - - await gather( - ops_test.model.relate(f"{database_application_name}:db", "indico1:db"), - ops_test.model.relate(f"{database_application_name}:db", "indico2:db"), - ) + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME, APPLICATION_NAME, f"{APPLICATION_NAME}2"], + status="active", + timeout=1000, + ) - await ops_test.model.wait_for_idle( - apps=[database_application_name], - status="blocked", - raise_on_blocked=False, - timeout=1000, - ) + await gather( + ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db"), + ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}2:db"), + ) - assert ( - ops_test.model.applications[database_application_name].units[0].workload_status_message - == EXTENSIONS_BLOCKING_MESSAGE - ) + leader_unit = await get_leader_unit(ops_test, DATABASE_APP_NAME) + await ops_test.model.block_until( + lambda: leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE, timeout=1000 + ) - await ops_test.model.applications[database_application_name].destroy_relation( - f"{database_application_name}:db", "indico1:db" - ) + assert leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE - await ops_test.model.wait_for_idle( - apps=[database_application_name], - status="blocked", - raise_on_blocked=False, - timeout=1000, - ) + logger.info("Verify that the charm remains blocked if there are other blocking relations") + await ops_test.model.applications[DATABASE_APP_NAME].destroy_relation( + f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db" + ) - # Verify that the charm remains blocked if there are other blocking relations - assert ( - ops_test.model.applications[database_application_name].units[0].workload_status_message - == EXTENSIONS_BLOCKING_MESSAGE - ) + await ops_test.model.block_until( + lambda: leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE, timeout=1000 + ) - await ops_test.model.applications[database_application_name].destroy_relation( - f"{database_application_name}:db", "indico2:db" - ) + assert leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE - # Verify that active status is restored when all blocking relations are gone - await ops_test.model.wait_for_idle( - apps=[database_application_name], - status="active", - raise_on_blocked=False, - timeout=1000, - ) + logger.info("Verify that active status is restored when all blocking relations are gone") + await ops_test.model.applications[DATABASE_APP_NAME].destroy_relation( + f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}2:db" + ) - # Verify that the charm doesn't block when the extensions are enabled. - logger.info("Verifying that the charm doesn't block when the extensions are enabled") - config = {"plugin_pg_trgm_enable": "True", "plugin_unaccent_enable": "True"} - await ops_test.model.applications[database_application_name].set_config(config) - await ops_test.model.wait_for_idle( - apps=[database_application_name], status="active", idle_period=15 - ) - await ops_test.model.relate(f"{database_application_name}:db", "indico1:db") - await ops_test.model.wait_for_idle( - apps=[database_application_name, "indico1"], - status="active", - raise_on_blocked=False, - timeout=2000, - ) + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME], + status="active", + timeout=1000, + ) - # Verify that the charm unblocks when the extensions are enabled after being blocked - # due to disabled extensions. - logger.info("Verifying that the charm unblocks when the extensions are enabled") - config = {"plugin_pg_trgm_enable": "False", "plugin_unaccent_enable": "False"} - await ops_test.model.applications[database_application_name].set_config(config) - await ops_test.model.applications[database_application_name].destroy_relation( - f"{database_application_name}:db", "indico1:db" - ) - wait_for_relation_removed_between(ops_test, database_application_name, "indico1") - await ops_test.model.wait_for_idle( - apps=[database_application_name, "indico1"], status="active", idle_period=15 - ) + logger.info("Verifying that the charm doesn't block when the extensions are enabled") + config = {"plugin_pg_trgm_enable": "True", "plugin_unaccent_enable": "True"} + await ops_test.model.applications[DATABASE_APP_NAME].set_config(config) + await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME], status="active") + await ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db") + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME, APPLICATION_NAME], + status="active", + timeout=2000, + ) - await ops_test.model.relate(f"{database_application_name}:db", "indico1:db") - unit = next(iter(ops_test.model.units.values())) - ops_test.model.block_until( - lambda: unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE, timeout=600 - ) + logger.info("Verifying that the charm unblocks when the extensions are enabled") + config = {"plugin_pg_trgm_enable": "False", "plugin_unaccent_enable": "False"} + await ops_test.model.applications[DATABASE_APP_NAME].set_config(config) + await ops_test.model.applications[DATABASE_APP_NAME].destroy_relation( + f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db" + ) + wait_for_relation_removed_between(ops_test, DATABASE_APP_NAME, APPLICATION_NAME) + await ops_test.model.wait_for_idle(apps=[DATABASE_APP_NAME, APPLICATION_NAME], status="active") - config = {"plugin_pg_trgm_enable": "True", "plugin_unaccent_enable": "True"} - await ops_test.model.applications[database_application_name].set_config(config) - await ops_test.model.wait_for_idle( - apps=[database_application_name, "indico1"], - status="active", - raise_on_blocked=False, - timeout=2000, - idle_period=15, - ) + await ops_test.model.relate(f"{DATABASE_APP_NAME}:db", f"{APPLICATION_NAME}:db") + ops_test.model.block_until( + lambda: leader_unit.workload_status_message == EXTENSIONS_BLOCKING_MESSAGE, timeout=1000 + ) - # Cleanup - await gather( - ops_test.model.remove_application("indico1", block_until_done=True), - ops_test.model.remove_application("indico2", block_until_done=True), - ops_test.model.remove_application("redis-broker", block_until_done=True), - ops_test.model.remove_application("redis-cache", block_until_done=True), - ) + config = {"plugin_pg_trgm_enable": "True", "plugin_unaccent_enable": "True"} + await ops_test.model.applications[DATABASE_APP_NAME].set_config(config) + await ops_test.model.wait_for_idle( + apps=[DATABASE_APP_NAME, APPLICATION_NAME], + status="active", + raise_on_blocked=False, + timeout=2000, + ) @pytest.mark.skip(reason="Should be ported and moved to the new relation tests")