From c93bbda8637e38d9d1263fedaf31d9ea3e0e3c23 Mon Sep 17 00:00:00 2001 From: Lucas Gameiro Borges Date: Wed, 26 Jun 2024 01:58:27 +0000 Subject: [PATCH] refactor test --- tests/integration/helpers.py | 221 +++++++++++++++++++++++++++ tests/integration/test_backups.py | 239 ++++-------------------------- 2 files changed, 248 insertions(+), 212 deletions(-) diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index 67fa1b59f2..f960d5d5e0 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -4,6 +4,7 @@ import asyncio import itertools import json +import logging import os import subprocess import tempfile @@ -37,6 +38,8 @@ STORAGE_PATH = METADATA["storage"]["pgdata"]["location"] APPLICATION_NAME = "postgresql-test-app" +logger = logging.getLogger(__name__) + async def build_connection_string( ops_test: OpsTest, @@ -1064,3 +1067,221 @@ def wait_for_relation_removed_between( break except RetryError: assert False, "Relation failed to exit after 3 minutes." + + +async def backup_operations( + ops_test: OpsTest, + s3_integrator_app_name: str, + tls_certificates_app_name: str, + tls_config, + tls_channel, + credentials, + cloud, + config, + charm, +) -> None: + # Deploy S3 Integrator and TLS Certificates Operator. + await ops_test.model.deploy(s3_integrator_app_name) + await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel) + + # Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now + # as archive_mode is disabled after restoring the backup) and to TLS Certificates Operator + # (to be able to create backups from replicas). + database_app_name = f"{DATABASE_APP_NAME}-{cloud.lower()}" + await ops_test.model.deploy( + charm, + application_name=database_app_name, + num_units=2, + series=CHARM_SERIES, + config={"profile": "testing"}, + ) + + await ops_test.model.relate(database_app_name, tls_certificates_app_name) + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle(apps=[database_app_name], status="active", timeout=1000) + await ops_test.model.relate(database_app_name, s3_integrator_app_name) + + # Configure and set access and secret keys. + logger.info(f"configuring S3 integrator for {cloud}") + await ops_test.model.applications[s3_integrator_app_name].set_config(config) + action = await ops_test.model.units.get(f"{s3_integrator_app_name}/0").run_action( + "sync-s3-credentials", + **credentials, + ) + await action.wait() + async with ops_test.fast_forward(fast_interval="60s"): + await ops_test.model.wait_for_idle( + apps=[database_app_name, s3_integrator_app_name], status="active", timeout=1500 + ) + + primary = await get_primary(ops_test, f"{database_app_name}/0") + for unit in ops_test.model.applications[database_app_name].units: + if unit.name != primary: + replica = unit.name + break + + # Write some data. + password = await get_password(ops_test, primary) + address = get_unit_address(ops_test, primary) + logger.info("creating a table in the database") + with db_connect(host=address, password=password) as connection: + connection.autocommit = True + connection.cursor().execute( + "CREATE TABLE IF NOT EXISTS backup_table_1 (test_collumn INT );" + ) + connection.close() + + # Run the "create backup" action. + logger.info("creating a backup") + action = await ops_test.model.units.get(replica).run_action("create-backup") + await action.wait() + backup_status = action.results.get("backup-status") + assert backup_status, "backup hasn't succeeded" + await ops_test.model.wait_for_idle( + apps=[database_app_name, s3_integrator_app_name], status="active", timeout=1000 + ) + + # With a stable cluster, Run the "create backup" action + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(status="active", timeout=1000, idle_period=30) + logger.info("listing the available backups") + action = await ops_test.model.units.get(replica).run_action("list-backups") + await action.wait() + backups = action.results.get("backups") + # 2 lines for header output, 1 backup line ==> 3 total lines + assert len(backups.split("\n")) == 3, "full backup is not outputted" + await ops_test.model.wait_for_idle(status="active", timeout=1000) + + # Write some data. + logger.info("creating a second table in the database") + with db_connect(host=address, password=password) as connection: + connection.autocommit = True + connection.cursor().execute("CREATE TABLE backup_table_2 (test_collumn INT );") + connection.close() + + # Run the "create backup" action. + logger.info("creating a backup") + action = await ops_test.model.units.get(replica).run_action( + "create-backup", **{"type": "differential"} + ) + await action.wait() + backup_status = action.results.get("backup-status") + assert backup_status, "backup hasn't succeeded" + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(status="active", timeout=1000) + + # Run the "list backups" action. + logger.info("listing the available backups") + action = await ops_test.model.units.get(replica).run_action("list-backups") + await action.wait() + backups = action.results.get("backups") + # 2 lines for header output, 2 backup lines ==> 4 total lines + assert len(backups.split("\n")) == 4, "differential backup is not outputted" + await ops_test.model.wait_for_idle(status="active", timeout=1000) + + # Write some data. + logger.info("creating a second table in the database") + with db_connect(host=address, password=password) as connection: + connection.autocommit = True + connection.cursor().execute("CREATE TABLE backup_table_3 (test_collumn INT );") + connection.close() + # Scale down to be able to restore. + async with ops_test.fast_forward(): + await ops_test.model.destroy_unit(replica) + await ops_test.model.block_until( + lambda: len(ops_test.model.applications[database_app_name].units) == 1 + ) + + for unit in ops_test.model.applications[database_app_name].units: + remaining_unit = unit + break + + # Run the "restore backup" action for differential backup. + for attempt in Retrying( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + logger.info("restoring the backup") + last_diff_backup = backups.split("\n")[-1] + backup_id = last_diff_backup.split()[0] + action = await remaining_unit.run_action("restore", **{"backup-id": backup_id}) + await action.wait() + restore_status = action.results.get("restore-status") + assert restore_status, "restore hasn't succeeded" + + # Wait for the restore to complete. + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(status="active", timeout=1000) + + # Check that the backup was correctly restored by having only the first created table. + logger.info("checking that the backup was correctly restored") + primary = await get_primary(ops_test, remaining_unit.name) + address = get_unit_address(ops_test, primary) + with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_1');" + ) + assert cursor.fetchone()[ + 0 + ], "backup wasn't correctly restored: table 'backup_table_1' doesn't exist" + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_2');" + ) + assert cursor.fetchone()[ + 0 + ], "backup wasn't correctly restored: table 'backup_table_2' doesn't exist" + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_3');" + ) + assert not cursor.fetchone()[ + 0 + ], "backup wasn't correctly restored: table 'backup_table_3' exists" + connection.close() + + # Run the "restore backup" action for full backup. + for attempt in Retrying( + stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) + ): + with attempt: + logger.info("restoring the backup") + last_full_backup = backups.split("\n")[-2] + backup_id = last_full_backup.split()[0] + action = await remaining_unit.run_action("restore", **{"backup-id": backup_id}) + await action.wait() + restore_status = action.results.get("restore-status") + assert restore_status, "restore hasn't succeeded" + + # Wait for the restore to complete. + async with ops_test.fast_forward(): + await ops_test.model.wait_for_idle(status="active", timeout=1000) + + # Check that the backup was correctly restored by having only the first created table. + primary = await get_primary(ops_test, remaining_unit.name) + address = get_unit_address(ops_test, primary) + logger.info("checking that the backup was correctly restored") + with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_1');" + ) + assert cursor.fetchone()[ + 0 + ], "backup wasn't correctly restored: table 'backup_table_1' doesn't exist" + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_2');" + ) + assert not cursor.fetchone()[ + 0 + ], "backup wasn't correctly restored: table 'backup_table_2' exists" + cursor.execute( + "SELECT EXISTS (SELECT FROM information_schema.tables" + " WHERE table_schema = 'public' AND table_name = 'backup_table_3');" + ) + assert not cursor.fetchone()[ + 0 + ], "backup wasn't correctly restored: table 'backup_table_3' exists" + connection.close() diff --git a/tests/integration/test_backups.py b/tests/integration/test_backups.py index 0425623351..3a00f0d5e0 100644 --- a/tests/integration/test_backups.py +++ b/tests/integration/test_backups.py @@ -14,6 +14,7 @@ from .helpers import ( CHARM_SERIES, DATABASE_APP_NAME, + backup_operations, construct_endpoint, db_connect, get_password, @@ -97,221 +98,23 @@ async def cloud_configs(ops_test: OpsTest, github_secrets) -> None: bucket_object.delete() -async def test_backup(ops_test: OpsTest, cloud, config, charm) -> None: - """Build and deploy two units of PostgreSQL and then test the backup and restore actions.""" - # Deploy S3 Integrator and TLS Certificates Operator. - await ops_test.model.deploy(S3_INTEGRATOR_APP_NAME) - await ops_test.model.deploy(tls_certificates_app_name, config=tls_config, channel=tls_channel) - - # Deploy and relate PostgreSQL to S3 integrator (one database app for each cloud for now - # as archive_mode is disabled after restoring the backup) and to TLS Certificates Operator - # (to be able to create backups from replicas). - database_app_name = f"{DATABASE_APP_NAME}-{cloud.lower()}" - await ops_test.model.deploy( - charm, - application_name=database_app_name, - num_units=2, - series=CHARM_SERIES, - config={"profile": "testing"}, - ) - - await ops_test.model.relate(database_app_name, tls_certificates_app_name) - async with ops_test.fast_forward(fast_interval="60s"): - await ops_test.model.wait_for_idle(apps=[database_app_name], status="active", timeout=1000) - await ops_test.model.relate(database_app_name, S3_INTEGRATOR_APP_NAME) - - # Configure and set access and secret keys. - logger.info(f"configuring S3 integrator for {cloud}") - await ops_test.model.applications[S3_INTEGRATOR_APP_NAME].set_config(config) - action = await ops_test.model.units.get(f"{S3_INTEGRATOR_APP_NAME}/0").run_action( - "sync-s3-credentials", - **cloud_configs[1][cloud], - ) - await action.wait() - async with ops_test.fast_forward(fast_interval="60s"): - await ops_test.model.wait_for_idle( - apps=[database_app_name, S3_INTEGRATOR_APP_NAME], status="active", timeout=1500 - ) - - primary = await get_primary(ops_test, f"{database_app_name}/0") - for unit in ops_test.model.applications[database_app_name].units: - if unit.name != primary: - replica = unit.name - break - - # Write some data. - password = await get_password(ops_test, primary) - address = get_unit_address(ops_test, primary) - logger.info("creating a table in the database") - with db_connect(host=address, password=password) as connection: - connection.autocommit = True - connection.cursor().execute( - "CREATE TABLE IF NOT EXISTS backup_table_1 (test_collumn INT );" - ) - connection.close() - - # Run the "create backup" action. - logger.info("creating a backup") - action = await ops_test.model.units.get(replica).run_action("create-backup") - await action.wait() - backup_status = action.results.get("backup-status") - assert backup_status, "backup hasn't succeeded" - await ops_test.model.wait_for_idle( - apps=[database_app_name, S3_INTEGRATOR_APP_NAME], status="active", timeout=1000 - ) - - # With a stable cluster, Run the "create backup" action - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(status="active", timeout=1000, idle_period=30) - logger.info("listing the available backups") - action = await ops_test.model.units.get(replica).run_action("list-backups") - await action.wait() - backups = action.results.get("backups") - # 2 lines for header output, 1 backup line ==> 3 total lines - assert len(backups.split("\n")) == 3, "full backup is not outputted" - await ops_test.model.wait_for_idle(status="active", timeout=1000) - - # Write some data. - logger.info("creating a second table in the database") - with db_connect(host=address, password=password) as connection: - connection.autocommit = True - connection.cursor().execute("CREATE TABLE backup_table_2 (test_collumn INT );") - connection.close() - - # Run the "create backup" action. - logger.info("creating a backup") - action = await ops_test.model.units.get(replica).run_action( - "create-backup", **{"type": "differential"} - ) - await action.wait() - backup_status = action.results.get("backup-status") - assert backup_status, "backup hasn't succeeded" - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(status="active", timeout=1000) - - # Run the "list backups" action. - logger.info("listing the available backups") - action = await ops_test.model.units.get(replica).run_action("list-backups") - await action.wait() - backups = action.results.get("backups") - # 2 lines for header output, 2 backup lines ==> 4 total lines - assert len(backups.split("\n")) == 4, "differential backup is not outputted" - await ops_test.model.wait_for_idle(status="active", timeout=1000) - - # Write some data. - logger.info("creating a second table in the database") - with db_connect(host=address, password=password) as connection: - connection.autocommit = True - connection.cursor().execute("CREATE TABLE backup_table_3 (test_collumn INT );") - connection.close() - # Scale down to be able to restore. - async with ops_test.fast_forward(): - await ops_test.model.destroy_unit(replica) - await ops_test.model.block_until( - lambda: len(ops_test.model.applications[database_app_name].units) == 1 - ) - - for unit in ops_test.model.applications[database_app_name].units: - remaining_unit = unit - break - - # Run the "restore backup" action for differential backup. - for attempt in Retrying( - stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) - ): - with attempt: - logger.info("restoring the backup") - last_diff_backup = backups.split("\n")[-1] - backup_id = last_diff_backup.split()[0] - action = await remaining_unit.run_action("restore", **{"backup-id": backup_id}) - await action.wait() - restore_status = action.results.get("restore-status") - assert restore_status, "restore hasn't succeeded" - - # Wait for the restore to complete. - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(status="active", timeout=1000) - - # Check that the backup was correctly restored by having only the first created table. - logger.info("checking that the backup was correctly restored") - primary = await get_primary(ops_test, remaining_unit.name) - address = get_unit_address(ops_test, primary) - with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_1');" - ) - assert cursor.fetchone()[ - 0 - ], "backup wasn't correctly restored: table 'backup_table_1' doesn't exist" - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_2');" - ) - assert cursor.fetchone()[ - 0 - ], "backup wasn't correctly restored: table 'backup_table_2' doesn't exist" - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_3');" - ) - assert not cursor.fetchone()[ - 0 - ], "backup wasn't correctly restored: table 'backup_table_3' exists" - connection.close() - - # Run the "restore backup" action for full backup. - for attempt in Retrying( - stop=stop_after_attempt(10), wait=wait_exponential(multiplier=1, min=2, max=30) - ): - with attempt: - logger.info("restoring the backup") - last_full_backup = backups.split("\n")[-2] - backup_id = last_full_backup.split()[0] - action = await remaining_unit.run_action("restore", **{"backup-id": backup_id}) - await action.wait() - restore_status = action.results.get("restore-status") - assert restore_status, "restore hasn't succeeded" - - # Wait for the restore to complete. - async with ops_test.fast_forward(): - await ops_test.model.wait_for_idle(status="active", timeout=1000) - - # Check that the backup was correctly restored by having only the first created table. - primary = await get_primary(ops_test, remaining_unit.name) - address = get_unit_address(ops_test, primary) - logger.info("checking that the backup was correctly restored") - with db_connect(host=address, password=password) as connection, connection.cursor() as cursor: - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_1');" - ) - assert cursor.fetchone()[ - 0 - ], "backup wasn't correctly restored: table 'backup_table_1' doesn't exist" - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_2');" - ) - assert not cursor.fetchone()[ - 0 - ], "backup wasn't correctly restored: table 'backup_table_2' exists" - cursor.execute( - "SELECT EXISTS (SELECT FROM information_schema.tables" - " WHERE table_schema = 'public' AND table_name = 'backup_table_3');" - ) - assert not cursor.fetchone()[ - 0 - ], "backup wasn't correctly restored: table 'backup_table_3' exists" - connection.close() - - @pytest.mark.group(1) @pytest.mark.abort_on_fail async def test_backup_aws(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict], charm) -> None: + """Build and deploy two units of PostgreSQL in AWS, test backup and restore actions.""" config = cloud_configs[0].get(AWS) - await test_backup(ops_test, AWS, config, charm) - + credentials = cloud_configs[1][AWS] + await backup_operations( + ops_test, + S3_INTEGRATOR_APP_NAME, + tls_certificates_app_name, + tls_config, + tls_channel, + credentials, + AWS, + config, + charm, + ) database_app_name = f"{DATABASE_APP_NAME}-aws" primary = await get_primary(ops_test, f"{database_app_name}/0") password = await get_password(ops_test, primary) @@ -375,9 +178,21 @@ async def test_backup_aws(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict], c @pytest.mark.group(2) @pytest.mark.abort_on_fail async def test_backup_gcp(ops_test: OpsTest, cloud_configs: Tuple[Dict, Dict], charm) -> None: + """Build and deploy two units of PostgreSQL in GCP, test backup and restore actions.""" config = cloud_configs[0].get(GCP) - await test_backup(ops_test, GCP, config, charm) + credentials = cloud_configs[1][GCP] + await backup_operations( + ops_test, + S3_INTEGRATOR_APP_NAME, + tls_certificates_app_name, + tls_config, + tls_channel, + credentials, + GCP, + config, + charm, + ) database_app_name = f"{DATABASE_APP_NAME}-gcp" # Remove the database app.