From f2afacec840ff49b6ebc723c5d56e59b5cab15f0 Mon Sep 17 00:00:00 2001
From: Lucian Petrut <petrutlucian94@gmail.com>
Date: Thu, 21 Nov 2024 00:12:26 +0200
Subject: [PATCH] Extend ceph sc tests (#174)

We'll extend the Ceph storage class test to actually create a PVC
and ensure that pods can read/write data from these volumes.

While at it, we're making some slight changes to the ceph bundle.
At the moment, it deploys a single unit with two OSDs and node based
replication (default). For this reason, the PGs are inactive and
the rbd commands hang (and implicitly the Ceph provisioner).
We'll fix this by using three units, each containing one OSD.

Co-authored-by: Adam Dyess <adam.dyess@canonical.com>
---
 tests/integration/conftest.py                 |  2 +-
 tests/integration/data/test-bundle-ceph.yaml  |  4 +-
 .../data/test_ceph/ceph-xfs-pvc.yaml          | 16 +++++
 .../data/test_ceph/pv-reader-pod.yaml         | 21 ++++++
 .../data/test_ceph/pv-writer-pod.yaml         | 21 ++++++
 tests/integration/helpers.py                  | 66 ++++++++++++++++++-
 tests/integration/test_ceph.py                | 40 ++++++++++-
 tox.ini                                       |  4 +-
 8 files changed, 168 insertions(+), 6 deletions(-)
 create mode 100755 tests/integration/data/test_ceph/ceph-xfs-pvc.yaml
 create mode 100755 tests/integration/data/test_ceph/pv-reader-pod.yaml
 create mode 100755 tests/integration/data/test_ceph/pv-writer-pod.yaml

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index ffd7857a..42a37073 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -343,7 +343,7 @@ async def deploy_model(
             await the_model.wait_for_idle(
                 apps=list(bundle.applications),
                 status="active",
-                timeout=30 * 60,
+                timeout=60 * 60,
             )
         try:
             yield the_model
diff --git a/tests/integration/data/test-bundle-ceph.yaml b/tests/integration/data/test-bundle-ceph.yaml
index b1a26b9a..8b803e9e 100644
--- a/tests/integration/data/test-bundle-ceph.yaml
+++ b/tests/integration/data/test-bundle-ceph.yaml
@@ -33,9 +33,9 @@ applications:
     charm: ceph-osd
     channel: quincy/stable
     constraints: cores=2 mem=4G root-disk=16G
-    num_units: 1
+    num_units: 3
     storage:
-      osd-devices: 1G,2
+      osd-devices: 1G,1
       osd-journals: 1G,1
 relations:
   - [k8s, k8s-worker:cluster]
diff --git a/tests/integration/data/test_ceph/ceph-xfs-pvc.yaml b/tests/integration/data/test_ceph/ceph-xfs-pvc.yaml
new file mode 100755
index 00000000..ae5c5685
--- /dev/null
+++ b/tests/integration/data/test_ceph/ceph-xfs-pvc.yaml
@@ -0,0 +1,16 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: raw-block-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: 64Mi
+  storageClassName: ceph-xfs
diff --git a/tests/integration/data/test_ceph/pv-reader-pod.yaml b/tests/integration/data/test_ceph/pv-reader-pod.yaml
new file mode 100755
index 00000000..c87c8691
--- /dev/null
+++ b/tests/integration/data/test_ceph/pv-reader-pod.yaml
@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pv-reader-test
+  namespace: default
+spec:
+  restartPolicy: Never
+  volumes:
+  - name: pvc-test
+    persistentVolumeClaim:
+      claimName: raw-block-pvc
+  containers:
+  - name: pv-reader
+    image: busybox
+    command: ["/bin/sh", "-c", "cat /pvc/test_file"]
+    volumeMounts:
+    - name: pvc-test
+      mountPath: /pvc
diff --git a/tests/integration/data/test_ceph/pv-writer-pod.yaml b/tests/integration/data/test_ceph/pv-writer-pod.yaml
new file mode 100755
index 00000000..129849b5
--- /dev/null
+++ b/tests/integration/data/test_ceph/pv-writer-pod.yaml
@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pv-writer-test
+  namespace: default
+spec:
+  restartPolicy: Never
+  volumes:
+  - name: pvc-test
+    persistentVolumeClaim:
+      claimName: raw-block-pvc
+  containers:
+  - name: pv-writer
+    image: busybox
+    command: ["/bin/sh", "-c", "echo 'PVC test data.' > /pvc/test_file"]
+    volumeMounts:
+    - name: pvc-test
+      mountPath: /pvc
diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py
index f58037e3..c1cee35a 100644
--- a/tests/integration/helpers.py
+++ b/tests/integration/helpers.py
@@ -2,6 +2,8 @@
 # See LICENSE file for licensing details.
 """Additions to tools missing from juju library."""
 
+# pylint: disable=too-many-arguments,too-many-positional-arguments
+
 import ipaddress
 import json
 import logging
@@ -9,8 +11,9 @@
 from typing import List
 
 import yaml
+from juju import unit
 from juju.model import Model
-from tenacity import retry, stop_after_attempt, wait_fixed
+from tenacity import AsyncRetrying, retry, stop_after_attempt, wait_fixed
 
 log = logging.getLogger(__name__)
 
@@ -117,3 +120,64 @@ async def ready_nodes(k8s, expected_count):
     for node, ready in ready_nodes.items():
         log.info("Node %s is %s..", node, "ready" if ready else "not ready")
         assert ready, f"Node not yet ready: {node}."
+
+
+async def wait_pod_phase(
+    k8s: unit.Unit,
+    name: str,
+    phase: str,
+    namespace: str = "default",
+    retry_times: int = 30,
+    retry_delay_s: int = 5,
+):
+    """Wait for the pod to reach the specified phase (e.g. Succeeded).
+
+    Args:
+        k8s: k8s unit
+        name: the pod name
+        phase: expected phase
+        namespace: pod namespace
+        retry_times: the number of retries
+        retry_delay_s: retry interval
+
+    """
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(retry_times), wait=wait_fixed(retry_delay_s)
+    ):
+        with attempt:
+            cmd = " ".join(
+                [
+                    "k8s kubectl wait",
+                    f"--namespace {namespace}",
+                    "--for=jsonpath='{.status.phase}'=" + phase,
+                    f"pod/{name}",
+                    "--timeout 1s",
+                ]
+            )
+            action = await k8s.run(cmd)
+            result = await action.wait()
+            assert (
+                result.results["return-code"] == 0
+            ), f"Failed waiting for pod to reach {phase} phase."
+
+
+async def get_pod_logs(
+    k8s: unit.Unit,
+    name: str,
+    namespace: str = "default",
+) -> str:
+    """Retrieve pod logs.
+
+    Args:
+        k8s: k8s unit
+        name: pod name
+        namespace: pod namespace
+
+    Returns:
+        the pod logs as string.
+    """
+    cmd = " ".join(["k8s kubectl logs", f"--namespace {namespace}", f"pod/{name}"])
+    action = await k8s.run(cmd)
+    result = await action.wait()
+    assert result.results["return-code"] == 0, f"Failed to retrieve pod {name} logs."
+    return result.results["stdout"]
diff --git a/tests/integration/test_ceph.py b/tests/integration/test_ceph.py
index c526a998..0bfc02d8 100644
--- a/tests/integration/test_ceph.py
+++ b/tests/integration/test_ceph.py
@@ -6,9 +6,13 @@
 # pylint: disable=duplicate-code
 """Integration tests."""
 
+from pathlib import Path
+
 import pytest
 from juju import model, unit
 
+from . import helpers
+
 # This pytest mark configures the test environment to use the Canonical Kubernetes
 # bundle with ceph, for all the test within this module.
 pytestmark = [
@@ -17,11 +21,45 @@
 ]
 
 
+def _get_data_file_path(name) -> str:
+    """Retrieve the full path of the specified test data file."""
+    path = Path(__file__).parent / "data" / "test_ceph" / name
+    return str(path)
+
+
 @pytest.mark.abort_on_fail
 async def test_ceph_sc(kubernetes_cluster: model.Model):
-    """Test that a ceph storage class is available."""
+    """Test that a ceph storage class is available and validate pv attachments."""
     k8s: unit.Unit = kubernetes_cluster.applications["k8s"].units[0]
     event = await k8s.run("k8s kubectl get sc -o=jsonpath='{.items[*].provisioner}'")
     result = await event.wait()
     stdout = result.results["stdout"]
     assert "rbd.csi.ceph.com" in stdout, f"No ceph provisioner found in: {stdout}"
+
+    # Copy pod definitions.
+    for fname in ["ceph-xfs-pvc.yaml", "pv-writer-pod.yaml", "pv-reader-pod.yaml"]:
+        await k8s.scp_to(_get_data_file_path(fname), f"/tmp/{fname}")
+
+    # Create "ceph-xfs" PVC.
+    event = await k8s.run("k8s kubectl apply -f /tmp/ceph-xfs-pvc.yaml")
+    result = await event.wait()
+    assert result.results["return-code"] == 0, "Failed to create pvc."
+
+    # Create a pod that writes to the Ceph PV.
+    event = await k8s.run("k8s kubectl apply -f /tmp/pv-writer-pod.yaml")
+    result = await event.wait()
+    assert result.results["return-code"] == 0, "Failed to create writer pod."
+
+    # Wait for the pod to exit successfully.
+    await helpers.wait_pod_phase(k8s, "pv-writer-test", "Succeeded")
+
+    # Create a pod that reads the PV data and writes it to the log.
+    event = await k8s.run("k8s kubectl apply -f /tmp/pv-reader-pod.yaml")
+    result = await event.wait()
+    assert result.results["return-code"] == 0, "Failed to create reader pod."
+
+    await helpers.wait_pod_phase(k8s, "pv-reader-test", "Succeeded")
+
+    # Check the logged PV data.
+    logs = await helpers.get_pod_logs(k8s, "pv-reader-test")
+    assert "PVC test data" in logs
diff --git a/tox.ini b/tox.ini
index 9241a42b..4ec8b7df 100644
--- a/tox.ini
+++ b/tox.ini
@@ -89,8 +89,10 @@ description = Run integration tests
 deps = -r test_requirements.txt
 commands =
     pytest -v --tb native \
-      --log-cli-level=INFO \
       -s {toxinidir}/tests/integration \
+      --log-cli-level INFO \
+      --log-format "%(asctime)s %(levelname)s %(message)s" \
+      --log-date-format "%Y-%m-%d %H:%M:%S" \
       --crash-dump=on-failure \
       --crash-dump-args='-j snap.k8s.* --as-root' \
       {posargs}