Add workload tracing integration tests (#427)

tests/integration/helpers.py

@@ -2,11 +2,17 @@
 # See LICENSE file for licensing details.
 
 import asyncio
+import json
 import logging
 from typing import Optional
 
+import requests
 import sh
+from juju.application import Application
+from juju.unit import Unit
+from minio import Minio
 from pytest_operator.plugin import OpsTest
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 logger = logging.getLogger(__name__)
 
@@ -94,3 +100,102 @@ def dequote(s: str):
     if isinstance(s, str) and s.startswith('"') and s.endswith('"'):
         s = s[1:-1]
     return s
+
+
+async def deploy_and_configure_minio(ops_test: OpsTest) -> None:
+    """Deploy and set up minio and s3-integrator needed for s3-like storage backend in the HA charms."""
+    config = {
+        "access-key": "accesskey",
+        "secret-key": "secretkey",
+    }
+    await ops_test.model.deploy("minio", channel="edge", trust=True, config=config)
+    await ops_test.model.wait_for_idle(apps=["minio"], status="active", timeout=2000)
+    minio_addr = await get_address(ops_test, "minio", 0)
+
+    mc_client = Minio(
+        f"{minio_addr}:9000",
+        access_key="accesskey",
+        secret_key="secretkey",
+        secure=False,
+    )
+
+    # create tempo bucket
+    found = mc_client.bucket_exists("tempo")
+    if not found:
+        mc_client.make_bucket("tempo")
+
+    # configure s3-integrator
+    s3_integrator_app: Application = ops_test.model.applications["s3-integrator"]
+    s3_integrator_leader: Unit = s3_integrator_app.units[0]
+
+    await s3_integrator_app.set_config(
+        {
+            "endpoint": f"minio-0.minio-endpoints.{ops_test.model.name}.svc.cluster.local:9000",
+            "bucket": "tempo",
+        }
+    )
+
+    action = await s3_integrator_leader.run_action("sync-s3-credentials", **config)
+    action_result = await action.wait()
+    assert action_result.status == "completed"
+
+
+async def deploy_tempo_cluster(ops_test: OpsTest):
+    """Deploys tempo in its HA version together with minio and s3-integrator."""
+    tempo_app = "tempo"
+    worker_app = "tempo-worker"
+    tempo_worker_charm_url, worker_channel = "tempo-worker-k8s", "edge"
+    tempo_coordinator_charm_url, coordinator_channel = "tempo-coordinator-k8s", "edge"
+    await ops_test.model.deploy(
+        tempo_worker_charm_url, application_name=worker_app, channel=worker_channel, trust=True
+    )
+    await ops_test.model.deploy(
+        tempo_coordinator_charm_url,
+        application_name=tempo_app,
+        channel=coordinator_channel,
+        trust=True,
+    )
+    await ops_test.model.deploy("s3-integrator", channel="edge")
+
+    await ops_test.model.integrate(tempo_app + ":s3", "s3-integrator" + ":s3-credentials")
+    await ops_test.model.integrate(tempo_app + ":tempo-cluster", worker_app + ":tempo-cluster")
+
+    await deploy_and_configure_minio(ops_test)
+    async with ops_test.fast_forward():
+        await ops_test.model.wait_for_idle(
+            apps=[tempo_app, worker_app, "s3-integrator"],
+            status="active",
+            timeout=2000,
+            idle_period=30,
+        )
+
+
+def get_traces(tempo_host: str, service_name="tracegen-otlp_http", tls=True):
+    """Get traces directly from Tempo REST API."""
+    url = f"{'https' if tls else 'http'}://{tempo_host}:3200/api/search?tags=service.name={service_name}"
+    req = requests.get(
+        url,
+        verify=False,
+    )
+    assert req.status_code == 200
+    traces = json.loads(req.text)["traces"]
+    return traces
+
+
+@retry(stop=stop_after_attempt(15), wait=wait_exponential(multiplier=1, min=4, max=10))
+async def get_traces_patiently(tempo_host, service_name="tracegen-otlp_http", tls=True):
+    """Get traces directly from Tempo REST API, but also try multiple times.
+
+    Useful for cases when Tempo might not return the traces immediately (its API is known for returning data in
+    random order).
+    """
+    traces = get_traces(tempo_host, service_name=service_name, tls=tls)
+    assert len(traces) > 0
+    return traces
+
+
+async def get_application_ip(ops_test: OpsTest, app_name: str) -> str:
+    """Get the application IP address."""
+    status = await ops_test.model.get_status()
+    app = status["applications"][app_name]
+    return app.public_address

tests/integration/test_workload_tracing.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+import logging
+from pathlib import Path
+
+import pytest
+import yaml
+from helpers import deploy_tempo_cluster, get_application_ip, get_traces_patiently
+
+logger = logging.getLogger(__name__)
+
+METADATA = yaml.safe_load(Path("./metadata.yaml").read_text())
+APP_NAME = "traefik"
+TEMPO_APP_NAME = "tempo"
+RESOURCES = {
+    "traefik-image": METADATA["resources"]["traefik-image"]["upstream-source"],
+}
+
+
+async def test_setup_env(ops_test):
+    await ops_test.model.set_config({"logging-config": "<root>=WARNING; unit=DEBUG"})
+
+
+@pytest.mark.abort_on_fail
+async def test_workload_tracing_is_present(ops_test, traefik_charm):
+    logger.info("deploying tempo cluster")
+    await deploy_tempo_cluster(ops_test)
+
+    logger.info("deploying local charm")
+    await ops_test.model.deploy(
+        traefik_charm, resources=RESOURCES, application_name=APP_NAME, trust=True
+    )
+    await ops_test.model.wait_for_idle(
+        apps=[APP_NAME], status="active", timeout=300, wait_for_exact_units=1
+    )
+
+    # we relate _only_ workload tracing not to confuse with charm traces
+    await ops_test.model.add_relation(
+        "{}:workload-tracing".format(APP_NAME), "{}:tracing".format(TEMPO_APP_NAME)
+    )
+    # but we also relate tempo to route through traefik so there's any traffic to generate traces from
+    await ops_test.model.add_relation(
+        "{}:ingress".format(TEMPO_APP_NAME), "{}:traefik-route".format(APP_NAME)
+    )
+    await ops_test.model.wait_for_idle(apps=[APP_NAME], status="active")
+
+    # Verify workload traces are ingested into Tempo
+    assert await get_traces_patiently(
+        await get_application_ip(ops_test, TEMPO_APP_NAME),
+        service_name=f"{APP_NAME}",
+        tls=False,
+    )

tox.ini

@@ -67,6 +67,7 @@ deps =
     # fix for https://github.com/jd/tenacity/issues/471
     tenacity==8.3.0
     sh
+    minio
     -r{toxinidir}/requirements.txt
 commands =
     pytest -v --tb native --log-cli-level=INFO -s {[vars]tst_path}/integration {posargs}