HA tests - Replication 1/2 (#42)

* provided model support

* remove workaround with latest ops

* method to discover cluster name

* replication tests done

* removed unused imports

* removed sneaky autoimport

* making the linter happy after tests with blue

* comment and extra assertion

* happy linter #35

requirements.txt

@@ -1,2 +1,2 @@
-ops >= 1.5.0
+ops >= 1.5.2
 tenacity==8.0.1

tests/integration/helpers.py

@@ -291,3 +291,44 @@ def instance_ip(model: str, instance: str) -> str:
     for line in output.decode("utf8").splitlines():
         if instance in line:
             return line.split()[2]
+
+
+async def app_name(ops_test: OpsTest) -> str:
+    """Returns the name of the application running MySQL.
+
+    This is important since not all deployments of the MySQL charm have the application name
+    "mysql".
+
+    Note: if multiple clusters are running MySQL this will return the one first found.
+    """
+    status = await ops_test.model.get_status()
+    for app in ops_test.model.applications:
+        # note that format of the charm field is not exactly "mysql" but instead takes the form
+        # of `local:focal/mysql-6`
+        if "mysql" in status["applications"][app]["charm"]:
+            return app
+
+    return None
+
+
+def cluster_name(unit: Unit, model_name: str) -> str:
+    """Returns the MySQL cluster name.
+
+    Args:
+        unit: A unit to get data from
+        model_name: The current model name
+    Returns:
+        The (str) mysql cluster name
+    """
+    output = subprocess.check_output(
+        [
+            "juju",
+            "show-unit",
+            unit.name,
+            "--format=json",
+            f"--model={model_name}",
+        ]
+    )
+    output = json.loads(output.decode("utf-8"))
+
+    return output[unit.name]["relation-info"][0]["application-data"]["cluster-name"]

tests/integration/test_charm.py → tests/integration/test_ha.py

@@ -12,6 +12,8 @@
 from pytest_operator.plugin import OpsTest
 
 from tests.integration.helpers import (
+    app_name,
+    cluster_name,
     execute_commands_on_unit,
     generate_random_string,
     get_primary_unit,
@@ -23,18 +25,28 @@
 
 METADATA = yaml.safe_load(Path("./metadata.yaml").read_text())
 APP_NAME = METADATA["name"]
-CLUSTER_NAME = "test_cluster"
+ANOTHER_APP_NAME = f"second{APP_NAME}"
 
 
 @pytest.mark.order(1)
 @pytest.mark.abort_on_fail
 @pytest.mark.ha_tests
 async def test_build_and_deploy(ops_test: OpsTest) -> None:
     """Build the charm and deploy 3 units to ensure a cluster is formed."""
+    if app := await app_name(ops_test):
+        if len(ops_test.model.applications[app].units) == 3:
+            return
+        else:
+            async with ops_test.fast_forward():
+                await scale_application(ops_test, app, 3)
+            return
+
     # Build and deploy charm from local source folder
     charm = await ops_test.build_charm(".")
-    config = {"cluster-name": CLUSTER_NAME}
-    await ops_test.model.deploy(charm, application_name=APP_NAME, config=config, num_units=3)
+    await ops_test.model.deploy(charm, application_name=APP_NAME, num_units=3)
+    # variable used to avoid rebuilding the charm
+    global another_charm
+    another_charm = charm
 
     # Reduce the update_status frequency until the cluster is deployed
     async with ops_test.fast_forward():
@@ -57,17 +69,21 @@ async def test_build_and_deploy(ops_test: OpsTest) -> None:
 @pytest.mark.order(2)
 @pytest.mark.abort_on_fail
 @pytest.mark.ha_tests
-async def test_consistent_data_replication_across_cluster(ops_test: OpsTest) -> None:
+async def test_consistent_data_replication_across_cluster(
+    ops_test: OpsTest,
+) -> None:
     """Confirm that data is replicated from the primary node to all the replicas."""
     # Insert values into a table on the primary unit
-    random_unit = ops_test.model.applications[APP_NAME].units[0]
+    app = await app_name(ops_test)
+    random_unit = ops_test.model.applications[app].units[0]
+    cluster = cluster_name(random_unit, ops_test.model.info.name)
     server_config_credentials = await get_server_config_credentials(random_unit)
 
     primary_unit = await get_primary_unit(
         ops_test,
         random_unit,
-        APP_NAME,
-        CLUSTER_NAME,
+        app,
+        cluster,
         server_config_credentials["username"],
         server_config_credentials["password"],
     )
@@ -96,7 +112,7 @@ async def test_consistent_data_replication_across_cluster(ops_test: OpsTest) ->
     ]
 
     # Confirm that the values are available on all units
-    for unit in ops_test.model.applications[APP_NAME].units:
+    for unit in ops_test.model.applications[app].units:
         unit_address = await unit.get_public_address()
 
         output = await execute_commands_on_unit(
@@ -113,18 +129,19 @@ async def test_consistent_data_replication_across_cluster(ops_test: OpsTest) ->
 @pytest.mark.ha_tests
 async def test_primary_reelection(ops_test: OpsTest) -> None:
     """Confirm that a new primary is elected when the current primary is torn down."""
-    await ops_test.model.set_config({"update-status-hook-interval": "10s"})
+    app = await app_name(ops_test)
 
-    application = ops_test.model.applications[APP_NAME]
+    application = ops_test.model.applications[app]
 
     random_unit = application.units[0]
+    cluster = cluster_name(random_unit, ops_test.model.info.name)
     server_config_credentials = await get_server_config_credentials(random_unit)
 
     primary_unit = await get_primary_unit(
         ops_test,
         random_unit,
-        APP_NAME,
-        CLUSTER_NAME,
+        app,
+        cluster,
         server_config_credentials["username"],
         server_config_credentials["password"],
     )
@@ -137,7 +154,7 @@ async def test_primary_reelection(ops_test: OpsTest) -> None:
     async with ops_test.fast_forward():
         await ops_test.model.block_until(lambda: len(application.units) == 2)
         await ops_test.model.wait_for_idle(
-            apps=[APP_NAME],
+            apps=[app],
             status="active",
             raise_on_blocked=True,
             timeout=1000,
@@ -148,8 +165,8 @@ async def test_primary_reelection(ops_test: OpsTest) -> None:
     new_primary_unit = await get_primary_unit(
         ops_test,
         random_unit,
-        APP_NAME,
-        CLUSTER_NAME,
+        app,
+        cluster,
         server_config_credentials["username"],
         server_config_credentials["password"],
     )
@@ -158,7 +175,7 @@ async def test_primary_reelection(ops_test: OpsTest) -> None:
 
     # Add the unit back and wait until it is active
     async with ops_test.fast_forward():
-        await scale_application(ops_test, APP_NAME, 3)
+        await scale_application(ops_test, app, 3)
 
 
 @pytest.mark.order(4)
@@ -167,16 +184,18 @@ async def test_primary_reelection(ops_test: OpsTest) -> None:
 async def test_cluster_preserves_data_on_delete(ops_test: OpsTest) -> None:
     """Test that data is preserved during scale up and scale down."""
     # Insert values into test table from the primary unit
-    application = ops_test.model.applications[APP_NAME]
+    app = await app_name(ops_test)
+    application = ops_test.model.applications[app]
 
     random_unit = application.units[0]
+    cluster = cluster_name(random_unit, ops_test.model.info.name)
     server_config_credentials = await get_server_config_credentials(random_unit)
 
     primary_unit = await get_primary_unit(
         ops_test,
         random_unit,
-        APP_NAME,
-        CLUSTER_NAME,
+        app,
+        cluster,
         server_config_credentials["username"],
         server_config_credentials["password"],
     )
@@ -197,11 +216,11 @@ async def test_cluster_preserves_data_on_delete(ops_test: OpsTest) -> None:
         commit=True,
     )
 
-    old_unit_names = [unit.name for unit in ops_test.model.applications[APP_NAME].units]
+    old_unit_names = [unit.name for unit in ops_test.model.applications[app].units]
 
     # Add a unit and wait until it is active
     async with ops_test.fast_forward():
-        await scale_application(ops_test, APP_NAME, 4)
+        await scale_application(ops_test, app, 4)
 
     added_unit = [unit for unit in application.units if unit.name not in old_unit_names][0]
 
@@ -223,11 +242,9 @@ async def test_cluster_preserves_data_on_delete(ops_test: OpsTest) -> None:
     # Destroy the recently created unit and wait until the application is active
     await ops_test.model.destroy_units(added_unit.name)
     async with ops_test.fast_forward():
-        await ops_test.model.block_until(
-            lambda: len(ops_test.model.applications[APP_NAME].units) == 3
-        )
+        await ops_test.model.block_until(lambda: len(ops_test.model.applications[app].units) == 3)
         await ops_test.model.wait_for_idle(
-            apps=[APP_NAME],
+            apps=[app],
             status="active",
             raise_on_blocked=True,
             timeout=1000,
@@ -243,3 +260,88 @@ async def test_cluster_preserves_data_on_delete(ops_test: OpsTest) -> None:
             select_data_sql,
         )
         assert random_chars in output
+
+
+@pytest.mark.order(5)
+@pytest.mark.ha_tests
+async def test_cluster_isolation(ops_test: OpsTest) -> None:
+    """Test for cluster data isolation.
+
+    This test creates a new cluster, create a new table on both cluster, write a single record with
+    the application name for each cluster, retrieve and compare these records, asserting they are
+    not the same.
+    """
+    app = await app_name(ops_test)
+    apps = [app, ANOTHER_APP_NAME]
+
+    # Build and deploy secondary charm
+    charm = another_charm or await ops_test.build_charm(".")
+    await ops_test.model.deploy(charm, application_name=ANOTHER_APP_NAME, num_units=1)
+    async with ops_test.fast_forward():
+        await ops_test.model.block_until(
+            lambda: len(ops_test.model.applications[ANOTHER_APP_NAME].units) == 1
+        )
+        await ops_test.model.wait_for_idle(
+            apps=[ANOTHER_APP_NAME],
+            status="active",
+            raise_on_blocked=True,
+            timeout=1000,
+        )
+
+    # retrieve connection data for each cluster
+    connection_data = dict()
+    for application in apps:
+        random_unit = ops_test.model.applications[application].units[0]
+        cluster = cluster_name(random_unit, ops_test.model.info.name)
+        server_config_credentials = await get_server_config_credentials(random_unit)
+        primary_unit = await get_primary_unit(
+            ops_test,
+            random_unit,
+            application,
+            cluster,
+            server_config_credentials["username"],
+            server_config_credentials["password"],
+        )
+
+        primary_unit_address = await primary_unit.get_public_address()
+
+        connection_data[application] = {
+            "host": primary_unit_address,
+            "username": server_config_credentials["username"],
+            "password": server_config_credentials["password"],
+        }
+
+    # write single distinct record to each cluster
+    for application in apps:
+        create_records_sql = [
+            "CREATE DATABASE IF NOT EXISTS test",
+            "DROP TABLE IF EXISTS test.cluster_isolation_table",
+            "CREATE TABLE test.cluster_isolation_table (id varchar(40), primary key(id))",
+            f"INSERT INTO test.cluster_isolation_table VALUES ('{application}')",
+        ]
+
+        await execute_commands_on_unit(
+            connection_data[application]["host"],
+            connection_data[application]["username"],
+            connection_data[application]["password"],
+            create_records_sql,
+            commit=True,
+        )
+
+    result = list()
+    # read single record from each cluster
+    for application in apps:
+        read_records_sql = ["SELECT id FROM test.cluster_isolation_table"]
+
+        output = await execute_commands_on_unit(
+            connection_data[application]["host"],
+            connection_data[application]["username"],
+            connection_data[application]["password"],
+            read_records_sql,
+            commit=False,
+        )
+
+        assert len(output) == 1, "Just one record must exist on the test table"
+        result.append(output[0])
+
+    assert result[0] != result[1], "Writes from one cluster are replicated to another cluster."

tests/unit/test_relation_mysql_legacy.py

@@ -2,9 +2,8 @@
 # See LICENSE file for licensing details.
 
 import unittest
-from unittest.mock import PropertyMock, patch
+from unittest.mock import patch
 
-from ops.charm import RelationDepartedEvent
 from ops.testing import Harness
 
 from charm import MySQLOperatorCharm
@@ -94,13 +93,5 @@ def test_maria_db_relation_departed(
         self.maria_db_relation_id = self.harness.add_relation(LEGACY_MYSQL, "other-app")
         self.harness.add_relation_unit(self.maria_db_relation_id, "other-app/0")
 
-        # Workaround ops.testing not setting `departing_unit` in v1.5.0
-        # ref https://github.com/canonical/operator/pull/790
-        maria_db_relation = self.charm.model.get_relation(LEGACY_MYSQL)
-        with patch.object(
-            RelationDepartedEvent, "departing_unit", new_callable=PropertyMock
-        ) as mock_departing_unit:
-            mock_departing_unit.return_value = list(maria_db_relation.units)[0]
-            self.harness.remove_relation(self.maria_db_relation_id)
-
+        self.harness.remove_relation(self.maria_db_relation_id)
         _delete_users_for_unit.assert_called_once_with("mysql-legacy-relation")

tests/unit/test_shared_db.py

@@ -2,10 +2,9 @@
 # See LICENSE file for licensing details.
 
 import unittest
-from unittest.mock import PropertyMock, patch
+from unittest.mock import patch
 
 from charms.mysql.v0.mysql import MySQLCreateApplicationDatabaseAndScopedUserError
-from ops.charm import RelationDepartedEvent
 from ops.model import BlockedStatus
 from ops.testing import Harness
 
@@ -134,13 +133,5 @@ def test_shared_db_relation_departed(
             },
         )
 
-        # Workaround ops.testing not setting `departing_unit` in v1.5.0
-        # ref https://github.com/canonical/operator/pull/790
-        shared_db_relation = self.charm.model.get_relation(LEGACY_DB_SHARED)
-        with patch.object(
-            RelationDepartedEvent, "departing_unit", new_callable=PropertyMock
-        ) as mock_departing_unit:
-            mock_departing_unit.return_value = list(shared_db_relation.units)[0]
-            self.harness.remove_relation_unit(self.shared_db_relation_id, "other-app/0")
-
+        self.harness.remove_relation_unit(self.shared_db_relation_id, "other-app/0")
         _delete_users_for_unit.assert_called_once_with("other-app/0")