From 467141989ef8d2494128d3cff7bb81eeef3e01da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20C=2E=20Mass=C3=B3n?= <939888+Abuelodelanada@users.noreply.github.com> Date: Thu, 9 Jun 2022 13:10:02 -0300 Subject: [PATCH] Add itests to check for data persistence (#299) * Add itests to check for data persistance * Update alertmanager lib * Update grafana_source lib * Update ingress_per_unit lib * Revert "Update ingress_per_unit lib" This reverts commit affc7af054c5ec78198e703eabd945ecfe727009. * Remove warning from itests * Remove test_kubectl_delete.py * Move test and improve variable names and comments * Linting... --- pyproject.toml | 5 +- tests/integration/test_kubectl_delete.py | 55 ---------------- .../test_remote_write_grafana_agent.py | 63 +++++++++++++++++++ tests/integration/test_upgrade_charm.py | 21 +++++++ 4 files changed, 88 insertions(+), 56 deletions(-) delete mode 100644 tests/integration/test_kubectl_delete.py diff --git a/pyproject.toml b/pyproject.toml index 5fc29d5b..d1c8e29d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,5 +54,8 @@ module = ["ops.*", "pytest.*", "pytest_operator.*", "prometheus_api_client.*", " ignore_missing_imports = true [[tool.mypy.overrides]] -module = ["charms.grafana_k8s.*"] +module = ["charms.grafana_k8s.*", "charms.alertmanager_k8s.*"] follow_imports = "silent" + +[tool.pytest.ini_options] +asyncio_mode = "auto" diff --git a/tests/integration/test_kubectl_delete.py b/tests/integration/test_kubectl_delete.py deleted file mode 100644 index 55f111b5..00000000 --- a/tests/integration/test_kubectl_delete.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2022 Canonical Ltd. -# See LICENSE file for licensing details. - - -import logging -from pathlib import Path - -import pytest -import yaml -from helpers import check_prometheus_is_ready - -logger = logging.getLogger(__name__) - -METADATA = yaml.safe_load(Path("./metadata.yaml").read_text()) -app_name = METADATA["name"] -resources = {"prometheus-image": METADATA["resources"]["prometheus-image"]["upstream-source"]} - -TIMEOUT = 60 - - -@pytest.mark.abort_on_fail -async def test_deploy_from_local_path(ops_test, prometheus_charm): - """Deploy the charm-under-test.""" - logger.debug("deploy local charm") - - # Need trust = True otherwise errors on ghwf (persistentvolumeclaims ... is forbidden) - await ops_test.model.deploy( - prometheus_charm, application_name=app_name, resources=resources, trust=True - ) - await ops_test.model.wait_for_idle(apps=[app_name], status="active", timeout=TIMEOUT) - assert await check_prometheus_is_ready(ops_test, app_name, 0) - - -@pytest.mark.abort_on_fail -async def test_kubectl_delete_pod(ops_test, prometheus_charm): - pod_name = f"{app_name}-0" - - cmd = [ - "sg", - "microk8s", - "-c", - " ".join(["microk8s.kubectl", "delete", "pod", "-n", ops_test.model_name, pod_name]), - ] - - logger.debug( - "Removing pod '%s' from model '%s' with cmd: %s", pod_name, ops_test.model_name, cmd - ) - - retcode, stdout, stderr = await ops_test.run(*cmd) - assert retcode == 0, f"kubectl failed: {(stderr or stdout).strip()}" - logger.debug(stdout) - await ops_test.model.block_until(lambda: len(ops_test.model.applications[app_name].units) > 0) - await ops_test.model.wait_for_idle(apps=[app_name], status="active", timeout=TIMEOUT) - assert await check_prometheus_is_ready(ops_test, app_name, 0) diff --git a/tests/integration/test_remote_write_grafana_agent.py b/tests/integration/test_remote_write_grafana_agent.py index 6397606f..d2ab3a83 100644 --- a/tests/integration/test_remote_write_grafana_agent.py +++ b/tests/integration/test_remote_write_grafana_agent.py @@ -9,6 +9,7 @@ from helpers import check_prometheus_is_ready, oci_image, run_promql logger = logging.getLogger(__name__) +prometheus_resources = {"prometheus-image": oci_image("./metadata.yaml", "prometheus-image")} @pytest.mark.abort_on_fail @@ -48,6 +49,68 @@ async def test_remote_write_with_grafana_agent(ops_test, prometheus_charm): ) +@pytest.mark.abort_on_fail +async def test_check_data_persist_on_kubectl_delete_pod(ops_test, prometheus_charm): + prometheus_app_name = "prometheus" + pod_name = f"{prometheus_app_name}-0" + query = "prometheus_tsdb_head_chunks_created_total{}" + total0 = await run_promql(ops_test, query, prometheus_app_name) + + # total0 is a list of dicts in which "value" is a list that contains + # the timestamp and the value itself. + num_head_chunks_before = int(total0[0]["value"][1]) + assert num_head_chunks_before > 0 + + cmd = [ + "sg", + "microk8s", + "-c", + " ".join(["microk8s.kubectl", "delete", "pod", "-n", ops_test.model_name, pod_name]), + ] + + logger.debug( + "Removing pod '%s' from model '%s' with cmd: %s", pod_name, ops_test.model_name, cmd + ) + + retcode, stdout, stderr = await ops_test.run(*cmd) + assert retcode == 0, f"kubectl failed: {(stderr or stdout).strip()}" + logger.debug(stdout) + + await ops_test.model.block_until( + lambda: len(ops_test.model.applications[prometheus_app_name].units) > 0 + ) + await ops_test.model.wait_for_idle(apps=[prometheus_app_name], status="active", timeout=60) + assert await check_prometheus_is_ready(ops_test, prometheus_app_name, 0) + + total1 = await run_promql(ops_test, query, prometheus_app_name) + num_head_chunks_after = int(total1[0]["value"][1]) + assert num_head_chunks_before <= num_head_chunks_after + + +@pytest.mark.abort_on_fail +async def test_check_data_not_persist_on_scale_0(ops_test, prometheus_charm): + prometheus_app_name = "prometheus" + + query = "prometheus_tsdb_head_chunks_created_total{}" + total0 = await run_promql(ops_test, query, prometheus_app_name) + + # total0 is a list of dicts in which "value" is a list that contains + # the timestamp and the value itself. + num_head_chunks_before = int(total0[0]["value"][1]) + + await ops_test.model.applications[prometheus_app_name].scale(scale_change=0) + await ops_test.model.block_until( + lambda: len(ops_test.model.applications[prometheus_app_name].units) == 0 + ) + await ops_test.model.applications[prometheus_app_name].scale(scale_change=1) + await ops_test.model.wait_for_idle(apps=[prometheus_app_name], status="active", timeout=120) + assert await check_prometheus_is_ready(ops_test, prometheus_app_name, 0) + + total1 = await run_promql(ops_test, query, prometheus_app_name) + num_head_chunks_after = int(total1[0]["value"][1]) + assert num_head_chunks_before <= num_head_chunks_after + + async def has_metric(ops_test, query: str, app_name: str) -> bool: # Throws if the query does not return any time series within 5 minutes, # and as a consequence, fails the test diff --git a/tests/integration/test_upgrade_charm.py b/tests/integration/test_upgrade_charm.py index c14f4600..3a73dc61 100644 --- a/tests/integration/test_upgrade_charm.py +++ b/tests/integration/test_upgrade_charm.py @@ -11,6 +11,7 @@ get_prometheus_rules, get_rules_for, oci_image, + run_promql, ) logger = logging.getLogger(__name__) @@ -71,3 +72,23 @@ async def test_rules_are_retained_after_upgrade(ops_test, prometheus_charm): rules_with_relation = await get_prometheus_rules(ops_test, prometheus_app_name, 0) tester_rules = get_rules_for(tester_app_name, rules_with_relation) assert len(tester_rules) == 1 + + +@pytest.mark.abort_on_fail +async def test_check_data_persist_on_charm_upgrade(ops_test, prometheus_charm): + query = "prometheus_tsdb_head_chunks_created_total{}" + total0 = await run_promql(ops_test, query, prometheus_app_name) + sum0 = int(total0[0]["value"][1]) + + logger.debug("upgrade deployed charm with local charm %s", prometheus_charm) + await ops_test.model.applications[prometheus_app_name].refresh( + path=prometheus_charm, resources=prometheus_resources + ) + await ops_test.model.wait_for_idle( + apps=app_names, status="active", timeout=300, idle_period=60 + ) + assert await check_prometheus_is_ready(ops_test, prometheus_app_name, 0) + + total1 = await run_promql(ops_test, query, prometheus_app_name) + sum1 = int(total1[0]["value"][1]) + assert sum0 <= sum1