Add tests covering multi-unit prometheus deployments (#274)

* Delete repeating tests
* Add upgrade, rescale tests
* Add tests for rescaling while upgrading
* Verify timeseries continuity

Co-authored-by: Ryan Barry <[email protected]>

.github/workflows/release-edge.yaml

@@ -13,7 +13,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
         with:
-          fetch-depth: 0  
+          fetch-depth: 0
       - name: Check libs
         uses: canonical/charming-actions/[email protected]
         with:

metadata.yaml

@@ -10,7 +10,7 @@ description: |
   supports aggregating high dimensional data and exposes a powerful
   query language PromQL. This charm deploys and operates Prometheus on
   Kubernetes clusters. Prometheus can raise alerts through a relation
-  with the Altermanager charm. Alerting rules for Prometheus need to
+  with the Alertmanager charm. Alerting rules for Prometheus need to
   be provided through a relation with the application that requires
   alerting. Prometheus provides its own dashboard for data
   visualization but a richer visualization interface may be obtained

tests/integration/helpers.py

@@ -4,6 +4,7 @@
 
 import logging
 from pathlib import Path
+from typing import List
 
 import yaml
 from pytest_operator.plugin import OpsTest
@@ -47,6 +48,22 @@ async def check_prometheus_is_ready(ops_test: OpsTest, app_name: str, unit_num:
     assert is_ready
 
 
+async def get_head_stats(ops_test: OpsTest, app_name: str, unit_num: int) -> dict:
+    """Get prometheus head stats.
+
+    Args:
+        ops_test: pytest-operator plugin
+        app_name: string name of Prometheus application
+        unit_num: integer number of a Prometheus juju unit
+
+    Returns:
+        A dict of headStats.
+    """
+    host = await unit_address(ops_test, app_name, unit_num)
+    prometheus = Prometheus(host=host)
+    return await prometheus.tsdb_head_stats()
+
+
 async def get_prometheus_config(ops_test: OpsTest, app_name: str, unit_num: int) -> str:
     """Fetch Prometheus configuration.
 
@@ -64,6 +81,25 @@ async def get_prometheus_config(ops_test: OpsTest, app_name: str, unit_num: int)
     return config
 
 
+async def get_prometheus_active_targets(
+    ops_test: OpsTest, app_name: str, unit_num: int
+) -> List[dict]:
+    """Fetch Prometheus active scrape targets.
+
+    Args:
+        ops_test: pytest-operator plugin
+        app_name: string name of Prometheus application
+        unit_num: integer number of a Prometheus juju unit
+
+    Returns:
+        Prometheus YAML configuration in string format.
+    """
+    host = await unit_address(ops_test, app_name, unit_num)
+    prometheus = Prometheus(host=host)
+    targets = await prometheus.active_targets()
+    return targets
+
+
 async def run_promql(ops_test: OpsTest, promql_query: str, app_name: str, unit_num: int = 0):
     """Run a PromQL query in Prometheus.
 
@@ -207,7 +243,6 @@ def remove_tester_alert_rule_file(name):
     """Remove an alert rule file from Prometheus Tester.
 
     Args:
-        rule: a string containing Prometheus alert rule in YAML format.
         name: a string name of alert rule file
     """
     rules_path = Path(TESTER_ALERT_RULES_PATH).joinpath(name)

tests/integration/prometheus-tester/src/charm.py

@@ -11,7 +11,7 @@
 from ops.charm import CharmBase
 from ops.main import main
 from ops.model import ActiveStatus, BlockedStatus
-from ops.pebble import Layer
+from ops.pebble import ChangeError, ExecError, Layer
 
 logger = logging.getLogger(__name__)
 
@@ -98,8 +98,24 @@ def _install_prometheus_client(self):
             return
 
         process = container.exec([self._pip_path, "install", "prometheus_client"])
-        process.wait()
-        logger.debug("Installed prometheus client")
+        try:
+            _, stderr = process.wait_output()
+            logger.debug("Installed prometheus client")
+            if stderr:
+                logger.warning(stderr)
+            return
+
+        except ExecError as e:
+            logger.error(
+                "Failed to install prometheus client: exited with code %d. Stderr:", e.exit_code
+            )
+            for line in e.stderr.splitlines():
+                logger.error("    %s", line)
+            self.unit.status = BlockedStatus("Failed to install prometheus client (see debug-log)")
+
+        except ChangeError as e:
+            logger.error("Failed to install prometheus client: %s", str(e))
+            self.unit.status = BlockedStatus("Failed to install prometheus client (see debug-log)")
 
     def _metrics_exporter(self):
         """Generate the metrics exporter script."""

tests/integration/test_charm.py

@@ -47,17 +47,7 @@ async def test_prometheus_scrape_relation_with_prometheus_tester(
         ),
     )
 
-    await ops_test.model.wait_for_idle(apps=app_names, status="active")
-
-    # TODO: Should not be needed.
-    #       Drop once https://github.com/juju/python-libjuju/issues/574 is resolved
-    #       - SA 2021-11-23
-    await ops_test.model.block_until(
-        lambda: (
-            len(ops_test.model.applications[prometheus_app_name].units) > 0
-            and len(ops_test.model.applications[tester_app_name].units) > 0
-        )
-    )
+    await ops_test.model.wait_for_idle(apps=app_names, status="active", wait_for_units=1)
 
     assert initial_workload_is_ready(ops_test, app_names)
     await check_prometheus_is_ready(ops_test, prometheus_app_name, 0)