Add itests to check for data persistence (#299)

* Add itests to check for data persistance

* Update alertmanager lib

* Update grafana_source lib

* Update ingress_per_unit lib

* Revert "Update ingress_per_unit lib"

This reverts commit affc7af.

* Remove warning from itests

* Remove test_kubectl_delete.py

* Move test and improve variable names and comments

* Linting...

pyproject.toml

@@ -54,5 +54,8 @@ module = ["ops.*", "pytest.*", "pytest_operator.*", "prometheus_api_client.*", "
 ignore_missing_imports = true
 
 [[tool.mypy.overrides]]
-module = ["charms.grafana_k8s.*"]
+module = ["charms.grafana_k8s.*", "charms.alertmanager_k8s.*"]
 follow_imports = "silent"
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

tests/integration/test_remote_write_grafana_agent.py

@@ -9,6 +9,7 @@
 from helpers import check_prometheus_is_ready, oci_image, run_promql
 
 logger = logging.getLogger(__name__)
+prometheus_resources = {"prometheus-image": oci_image("./metadata.yaml", "prometheus-image")}
 
 
 @pytest.mark.abort_on_fail
@@ -48,6 +49,68 @@ async def test_remote_write_with_grafana_agent(ops_test, prometheus_charm):
     )
 
 
+@pytest.mark.abort_on_fail
+async def test_check_data_persist_on_kubectl_delete_pod(ops_test, prometheus_charm):
+    prometheus_app_name = "prometheus"
+    pod_name = f"{prometheus_app_name}-0"
+    query = "prometheus_tsdb_head_chunks_created_total{}"
+    total0 = await run_promql(ops_test, query, prometheus_app_name)
+
+    # total0 is a list of dicts in which "value" is a list that contains
+    # the timestamp and the value itself.
+    num_head_chunks_before = int(total0[0]["value"][1])
+    assert num_head_chunks_before > 0
+
+    cmd = [
+        "sg",
+        "microk8s",
+        "-c",
+        " ".join(["microk8s.kubectl", "delete", "pod", "-n", ops_test.model_name, pod_name]),
+    ]
+
+    logger.debug(
+        "Removing pod '%s' from model '%s' with cmd: %s", pod_name, ops_test.model_name, cmd
+    )
+
+    retcode, stdout, stderr = await ops_test.run(*cmd)
+    assert retcode == 0, f"kubectl failed: {(stderr or stdout).strip()}"
+    logger.debug(stdout)
+
+    await ops_test.model.block_until(
+        lambda: len(ops_test.model.applications[prometheus_app_name].units) > 0
+    )
+    await ops_test.model.wait_for_idle(apps=[prometheus_app_name], status="active", timeout=60)
+    assert await check_prometheus_is_ready(ops_test, prometheus_app_name, 0)
+
+    total1 = await run_promql(ops_test, query, prometheus_app_name)
+    num_head_chunks_after = int(total1[0]["value"][1])
+    assert num_head_chunks_before <= num_head_chunks_after
+
+
+@pytest.mark.abort_on_fail
+async def test_check_data_not_persist_on_scale_0(ops_test, prometheus_charm):
+    prometheus_app_name = "prometheus"
+
+    query = "prometheus_tsdb_head_chunks_created_total{}"
+    total0 = await run_promql(ops_test, query, prometheus_app_name)
+
+    # total0 is a list of dicts in which "value" is a list that contains
+    # the timestamp and the value itself.
+    num_head_chunks_before = int(total0[0]["value"][1])
+
+    await ops_test.model.applications[prometheus_app_name].scale(scale_change=0)
+    await ops_test.model.block_until(
+        lambda: len(ops_test.model.applications[prometheus_app_name].units) == 0
+    )
+    await ops_test.model.applications[prometheus_app_name].scale(scale_change=1)
+    await ops_test.model.wait_for_idle(apps=[prometheus_app_name], status="active", timeout=120)
+    assert await check_prometheus_is_ready(ops_test, prometheus_app_name, 0)
+
+    total1 = await run_promql(ops_test, query, prometheus_app_name)
+    num_head_chunks_after = int(total1[0]["value"][1])
+    assert num_head_chunks_before <= num_head_chunks_after
+
+
 async def has_metric(ops_test, query: str, app_name: str) -> bool:
     # Throws if the query does not return any time series within 5 minutes,
     # and as a consequence, fails the test

tests/integration/test_upgrade_charm.py

@@ -11,6 +11,7 @@
     get_prometheus_rules,
     get_rules_for,
     oci_image,
+    run_promql,
 )
 
 logger = logging.getLogger(__name__)
@@ -71,3 +72,23 @@ async def test_rules_are_retained_after_upgrade(ops_test, prometheus_charm):
     rules_with_relation = await get_prometheus_rules(ops_test, prometheus_app_name, 0)
     tester_rules = get_rules_for(tester_app_name, rules_with_relation)
     assert len(tester_rules) == 1
+
+
+@pytest.mark.abort_on_fail
+async def test_check_data_persist_on_charm_upgrade(ops_test, prometheus_charm):
+    query = "prometheus_tsdb_head_chunks_created_total{}"
+    total0 = await run_promql(ops_test, query, prometheus_app_name)
+    sum0 = int(total0[0]["value"][1])
+
+    logger.debug("upgrade deployed charm with local charm %s", prometheus_charm)
+    await ops_test.model.applications[prometheus_app_name].refresh(
+        path=prometheus_charm, resources=prometheus_resources
+    )
+    await ops_test.model.wait_for_idle(
+        apps=app_names, status="active", timeout=300, idle_period=60
+    )
+    assert await check_prometheus_is_ready(ops_test, prometheus_app_name, 0)
+
+    total1 = await run_promql(ops_test, query, prometheus_app_name)
+    sum1 = int(total1[0]["value"][1])
+    assert sum0 <= sum1