From 2a0a3d3eb95699cb2889907d2581429a62593f8a Mon Sep 17 00:00:00 2001 From: Dragomir Penev Date: Fri, 10 Nov 2023 11:49:59 +0200 Subject: [PATCH] Revert backup changes --- .github/workflows/ci.yaml | 6 +++-- poetry.lock | 18 +++++++++++++- pyproject.toml | 1 + tests/integration/test_backups.py | 41 +++++++++++-------------------- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a764be08d8..1f266b7d0f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -50,6 +50,7 @@ jobs: name: Integration test charm on Juju 3 needs: - build + - lint uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@test-large with: artifact-name: ${{ needs.build.outputs.artifact-name }} @@ -62,13 +63,14 @@ jobs: "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}", "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}", "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}", - "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}" + "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}", } integration-test-juju2: name: Integration test charm on Juju 2 needs: - build + - lint uses: canonical/data-platform-workflows/.github/workflows/integration_test_charm.yaml@test-large with: artifact-name: ${{ needs.build.outputs.artifact-name }} @@ -82,5 +84,5 @@ jobs: "AWS_ACCESS_KEY": "${{ secrets.AWS_ACCESS_KEY }}", "AWS_SECRET_KEY": "${{ secrets.AWS_SECRET_KEY }}", "GCP_ACCESS_KEY": "${{ secrets.GCP_ACCESS_KEY }}", - "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}" + "GCP_SECRET_KEY": "${{ secrets.GCP_SECRET_KEY }}", } diff --git a/poetry.lock b/poetry.lock index 3f4d3936bf..34f28a62d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1457,6 +1457,22 @@ pytest = ">=7.0.0" docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] +[[package]] +name = "pytest-github-secrets" +version = "0.1.0" +description = "" +optional = false +python-versions = "^3.8" +files = [] +develop = false + +[package.source] +type = "git" +url = "https://github.com/canonical/data-platform-workflows" +reference = "v5.1.2" +resolved_reference = "3cc668dc10fa7316da9600c296ca7640d7d83222" +subdirectory = "python/pytest_plugins/github_secrets" + [[package]] name = "pytest-operator" version = "0.29.0" @@ -2042,4 +2058,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10.6" -content-hash = "73c1278c20dd2fa473126233af7f98f41676b11bb91773a0ea03e7564526f54f" +content-hash = "f85d4726f6baa77fe06879b2cf2e1231a2449da56ca6ca41a3542b6f4701c609" diff --git a/pyproject.toml b/pyproject.toml index 027bc6a289..aa6892a8d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ optional = true [tool.poetry.group.integration.dependencies] pytest = "^7.4.0" +pytest-github-secrets = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.1.2", subdirectory = "python/pytest_plugins/github_secrets"} pytest-operator = "^0.29.0" pytest-operator-cache = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.1.2", subdirectory = "python/pytest_plugins/pytest_operator_cache"} pytest-operator-groups = {git = "https://github.com/canonical/data-platform-workflows", tag = "v5.1.2", subdirectory = "python/pytest_plugins/pytest_operator_groups"} diff --git a/tests/integration/test_backups.py b/tests/integration/test_backups.py index 74f12249a3..b8e474cd9a 100644 --- a/tests/integration/test_backups.py +++ b/tests/integration/test_backups.py @@ -1,9 +1,7 @@ #!/usr/bin/env python3 # Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. -import json import logging -import os import uuid from typing import Dict, Tuple @@ -38,9 +36,8 @@ @pytest.fixture(scope="module") -async def cloud_configs(ops_test: OpsTest) -> None: +async def cloud_configs(ops_test: OpsTest, github_secrets) -> None: # Define some configurations and credentials. - secrets = json.loads(os.environ.get("SECRETS_FROM_GITHUB", "{}")) configs = { AWS: { "endpoint": "https://s3.amazonaws.com", @@ -57,19 +54,17 @@ async def cloud_configs(ops_test: OpsTest) -> None: } credentials = { AWS: { - "access-key": secrets.get("AWS_ACCESS_KEY"), - "secret-key": secrets.get("AWS_SECRET_KEY"), + "access-key": github_secrets["AWS_ACCESS_KEY"], + "secret-key": github_secrets["AWS_SECRET_KEY"], }, GCP: { - "access-key": secrets.get("GCP_ACCESS_KEY"), - "secret-key": secrets.get("GCP_SECRET_KEY"), + "access-key": github_secrets["GCP_ACCESS_KEY"], + "secret-key": github_secrets["GCP_SECRET_KEY"], }, } yield configs, credentials # Delete the previously created objects. for cloud, config in configs.items(): - if not credentials[cloud].get("secret-key"): - continue session = boto3.session.Session( aws_access_key_id=credentials[cloud]["access-key"], aws_secret_access_key=credentials[cloud]["secret-key"], @@ -84,7 +79,12 @@ async def cloud_configs(ops_test: OpsTest) -> None: bucket_object.delete() -@pytest.mark.group(1) +async def test_none() -> None: + """Empty test so that the suite will not fail if all tests are skippedi.""" + pass + + +@pytest.mark.uses_secrets @pytest.mark.abort_on_fail async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> None: """Build and deploy two units of PostgreSQL and then test the backup and restore actions.""" @@ -97,11 +97,6 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> No await ops_test.model.deploy(TLS_CERTIFICATES_APP_NAME, config=config, channel="legacy/stable") for cloud, config in cloud_configs[0].items(): - # Check if credentials are available - if not cloud_configs[1][cloud]["secret-key"]: - logger.warning(f"Skipping tests for {cloud}. No credentials provided") - continue - # Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now # as archive_mode is disabled after restoring the backup) and to TLS Certificates Operator # (to be able to create backups from replicas). @@ -125,7 +120,7 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> No ) await action.wait() await ops_test.model.wait_for_idle( - apps=[database_app_name, S3_INTEGRATOR_APP_NAME], status="active", timeout=1500 + apps=[database_app_name, S3_INTEGRATOR_APP_NAME], status="active", timeout=1000 ) primary = await get_primary(ops_test, f"{database_app_name}/0") @@ -227,12 +222,9 @@ async def test_backup(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> No await ops_test.model.remove_application(TLS_CERTIFICATES_APP_NAME, block_until_done=True) -@pytest.mark.group(1) -async def test_restore_on_new_cluster(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict]) -> None: +@pytest.mark.uses_secrets +async def test_restore_on_new_cluster(ops_test: OpsTest) -> None: """Test that is possible to restore a backup to another PostgreSQL cluster.""" - if not cloud_configs[1][AWS]["secret-key"]: - pytest.skip("No AWS credentials") - charm = await ops_test.build_charm(".") database_app_name = f"new-{DATABASE_APP_NAME}" await ops_test.model.deploy( @@ -308,14 +300,11 @@ async def test_restore_on_new_cluster(ops_test: OpsTest, cloud_configs: Tuple[Di connection.close() -@pytest.mark.group(1) +@pytest.mark.uses_secrets async def test_invalid_config_and_recovery_after_fixing_it( ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict] ) -> None: """Test that the charm can handle invalid and valid backup configurations.""" - if not cloud_configs[1][AWS]["secret-key"]: - pytest.skip("No AWS credentials") - database_app_name = f"new-{DATABASE_APP_NAME}" # Provide invalid backup configurations.