Extend ceph sc tests (#174)

We'll extend the Ceph storage class test to actually create a PVC
and ensure that pods can read/write data from these volumes.

While at it, we're making some slight changes to the ceph bundle.
At the moment, it deploys a single unit with two OSDs and node based
replication (default). For this reason, the PGs are inactive and
the rbd commands hang (and implicitly the Ceph provisioner).
We'll fix this by using three units, each containing one OSD.

Co-authored-by: Adam Dyess <[email protected]>

tests/integration/conftest.py

@@ -343,7 +343,7 @@ async def deploy_model(
             await the_model.wait_for_idle(
                 apps=list(bundle.applications),
                 status="active",
-                timeout=30 * 60,
+                timeout=60 * 60,
             )
         try:
             yield the_model

tests/integration/data/test-bundle-ceph.yaml

@@ -33,9 +33,9 @@ applications:
     charm: ceph-osd
     channel: quincy/stable
     constraints: cores=2 mem=4G root-disk=16G
-    num_units: 1
+    num_units: 3
     storage:
-      osd-devices: 1G,2
+      osd-devices: 1G,1
       osd-journals: 1G,1
 relations:
   - [k8s, k8s-worker:cluster]

tests/integration/data/test_ceph/ceph-xfs-pvc.yaml

@@ -0,0 +1,16 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+apiVersion: v1
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: raw-block-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Filesystem
+  resources:
+    requests:
+      storage: 64Mi
+  storageClassName: ceph-xfs

tests/integration/data/test_ceph/pv-reader-pod.yaml

@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pv-reader-test
+  namespace: default
+spec:
+  restartPolicy: Never
+  volumes:
+  - name: pvc-test
+    persistentVolumeClaim:
+      claimName: raw-block-pvc
+  containers:
+  - name: pv-reader
+    image: busybox
+    command: ["/bin/sh", "-c", "cat /pvc/test_file"]
+    volumeMounts:
+    - name: pvc-test
+      mountPath: /pvc

tests/integration/data/test_ceph/pv-writer-pod.yaml

@@ -0,0 +1,21 @@
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pv-writer-test
+  namespace: default
+spec:
+  restartPolicy: Never
+  volumes:
+  - name: pvc-test
+    persistentVolumeClaim:
+      claimName: raw-block-pvc
+  containers:
+  - name: pv-writer
+    image: busybox
+    command: ["/bin/sh", "-c", "echo 'PVC test data.' > /pvc/test_file"]
+    volumeMounts:
+    - name: pvc-test
+      mountPath: /pvc

tests/integration/helpers.py

@@ -2,15 +2,18 @@
 # See LICENSE file for licensing details.
 """Additions to tools missing from juju library."""
 
+# pylint: disable=too-many-arguments,too-many-positional-arguments
+
 import ipaddress
 import json
 import logging
 from pathlib import Path
 from typing import List
 
 import yaml
+from juju import unit
 from juju.model import Model
-from tenacity import retry, stop_after_attempt, wait_fixed
+from tenacity import AsyncRetrying, retry, stop_after_attempt, wait_fixed
 
 log = logging.getLogger(__name__)
 
@@ -117,3 +120,64 @@ async def ready_nodes(k8s, expected_count):
     for node, ready in ready_nodes.items():
         log.info("Node %s is %s..", node, "ready" if ready else "not ready")
         assert ready, f"Node not yet ready: {node}."
+
+
+async def wait_pod_phase(
+    k8s: unit.Unit,
+    name: str,
+    phase: str,
+    namespace: str = "default",
+    retry_times: int = 30,
+    retry_delay_s: int = 5,
+):
+    """Wait for the pod to reach the specified phase (e.g. Succeeded).
+
+    Args:
+        k8s: k8s unit
+        name: the pod name
+        phase: expected phase
+        namespace: pod namespace
+        retry_times: the number of retries
+        retry_delay_s: retry interval
+
+    """
+    async for attempt in AsyncRetrying(
+        stop=stop_after_attempt(retry_times), wait=wait_fixed(retry_delay_s)
+    ):
+        with attempt:
+            cmd = " ".join(
+                [
+                    "k8s kubectl wait",
+                    f"--namespace {namespace}",
+                    "--for=jsonpath='{.status.phase}'=" + phase,
+                    f"pod/{name}",
+                    "--timeout 1s",
+                ]
+            )
+            action = await k8s.run(cmd)
+            result = await action.wait()
+            assert (
+                result.results["return-code"] == 0
+            ), f"Failed waiting for pod to reach {phase} phase."
+
+
+async def get_pod_logs(
+    k8s: unit.Unit,
+    name: str,
+    namespace: str = "default",
+) -> str:
+    """Retrieve pod logs.
+
+    Args:
+        k8s: k8s unit
+        name: pod name
+        namespace: pod namespace
+
+    Returns:
+        the pod logs as string.
+    """
+    cmd = " ".join(["k8s kubectl logs", f"--namespace {namespace}", f"pod/{name}"])
+    action = await k8s.run(cmd)
+    result = await action.wait()
+    assert result.results["return-code"] == 0, f"Failed to retrieve pod {name} logs."
+    return result.results["stdout"]

tests/integration/test_ceph.py

@@ -6,9 +6,13 @@
 # pylint: disable=duplicate-code
 """Integration tests."""
 
+from pathlib import Path
+
 import pytest
 from juju import model, unit
 
+from . import helpers
+
 # This pytest mark configures the test environment to use the Canonical Kubernetes
 # bundle with ceph, for all the test within this module.
 pytestmark = [
@@ -17,11 +21,45 @@
 ]
 
 
+def _get_data_file_path(name) -> str:
+    """Retrieve the full path of the specified test data file."""
+    path = Path(__file__).parent / "data" / "test_ceph" / name
+    return str(path)
+
+
 @pytest.mark.abort_on_fail
 async def test_ceph_sc(kubernetes_cluster: model.Model):
-    """Test that a ceph storage class is available."""
+    """Test that a ceph storage class is available and validate pv attachments."""
     k8s: unit.Unit = kubernetes_cluster.applications["k8s"].units[0]
     event = await k8s.run("k8s kubectl get sc -o=jsonpath='{.items[*].provisioner}'")
     result = await event.wait()
     stdout = result.results["stdout"]
     assert "rbd.csi.ceph.com" in stdout, f"No ceph provisioner found in: {stdout}"
+
+    # Copy pod definitions.
+    for fname in ["ceph-xfs-pvc.yaml", "pv-writer-pod.yaml", "pv-reader-pod.yaml"]:
+        await k8s.scp_to(_get_data_file_path(fname), f"/tmp/{fname}")
+
+    # Create "ceph-xfs" PVC.
+    event = await k8s.run("k8s kubectl apply -f /tmp/ceph-xfs-pvc.yaml")
+    result = await event.wait()
+    assert result.results["return-code"] == 0, "Failed to create pvc."
+
+    # Create a pod that writes to the Ceph PV.
+    event = await k8s.run("k8s kubectl apply -f /tmp/pv-writer-pod.yaml")
+    result = await event.wait()
+    assert result.results["return-code"] == 0, "Failed to create writer pod."
+
+    # Wait for the pod to exit successfully.
+    await helpers.wait_pod_phase(k8s, "pv-writer-test", "Succeeded")
+
+    # Create a pod that reads the PV data and writes it to the log.
+    event = await k8s.run("k8s kubectl apply -f /tmp/pv-reader-pod.yaml")
+    result = await event.wait()
+    assert result.results["return-code"] == 0, "Failed to create reader pod."
+
+    await helpers.wait_pod_phase(k8s, "pv-reader-test", "Succeeded")
+
+    # Check the logged PV data.
+    logs = await helpers.get_pod_logs(k8s, "pv-reader-test")
+    assert "PVC test data" in logs

tox.ini

@@ -89,8 +89,10 @@ description = Run integration tests
 deps = -r test_requirements.txt
 commands =
     pytest -v --tb native \
-      --log-cli-level=INFO \
       -s {toxinidir}/tests/integration \
+      --log-cli-level INFO \
+      --log-format "%(asctime)s %(levelname)s %(message)s" \
+      --log-date-format "%Y-%m-%d %H:%M:%S" \
       --crash-dump=on-failure \
       --crash-dump-args='-j snap.k8s.* --as-root' \
       {posargs}