From f2afacec840ff49b6ebc723c5d56e59b5cab15f0 Mon Sep 17 00:00:00 2001 From: Lucian Petrut <petrutlucian94@gmail.com> Date: Thu, 21 Nov 2024 00:12:26 +0200 Subject: [PATCH] Extend ceph sc tests (#174) We'll extend the Ceph storage class test to actually create a PVC and ensure that pods can read/write data from these volumes. While at it, we're making some slight changes to the ceph bundle. At the moment, it deploys a single unit with two OSDs and node based replication (default). For this reason, the PGs are inactive and the rbd commands hang (and implicitly the Ceph provisioner). We'll fix this by using three units, each containing one OSD. Co-authored-by: Adam Dyess <adam.dyess@canonical.com> --- tests/integration/conftest.py | 2 +- tests/integration/data/test-bundle-ceph.yaml | 4 +- .../data/test_ceph/ceph-xfs-pvc.yaml | 16 +++++ .../data/test_ceph/pv-reader-pod.yaml | 21 ++++++ .../data/test_ceph/pv-writer-pod.yaml | 21 ++++++ tests/integration/helpers.py | 66 ++++++++++++++++++- tests/integration/test_ceph.py | 40 ++++++++++- tox.ini | 4 +- 8 files changed, 168 insertions(+), 6 deletions(-) create mode 100755 tests/integration/data/test_ceph/ceph-xfs-pvc.yaml create mode 100755 tests/integration/data/test_ceph/pv-reader-pod.yaml create mode 100755 tests/integration/data/test_ceph/pv-writer-pod.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index ffd7857a..42a37073 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -343,7 +343,7 @@ async def deploy_model( await the_model.wait_for_idle( apps=list(bundle.applications), status="active", - timeout=30 * 60, + timeout=60 * 60, ) try: yield the_model diff --git a/tests/integration/data/test-bundle-ceph.yaml b/tests/integration/data/test-bundle-ceph.yaml index b1a26b9a..8b803e9e 100644 --- a/tests/integration/data/test-bundle-ceph.yaml +++ b/tests/integration/data/test-bundle-ceph.yaml @@ -33,9 +33,9 @@ applications: charm: ceph-osd channel: quincy/stable constraints: cores=2 mem=4G root-disk=16G - num_units: 1 + num_units: 3 storage: - osd-devices: 1G,2 + osd-devices: 1G,1 osd-journals: 1G,1 relations: - [k8s, k8s-worker:cluster] diff --git a/tests/integration/data/test_ceph/ceph-xfs-pvc.yaml b/tests/integration/data/test_ceph/ceph-xfs-pvc.yaml new file mode 100755 index 00000000..ae5c5685 --- /dev/null +++ b/tests/integration/data/test_ceph/ceph-xfs-pvc.yaml @@ -0,0 +1,16 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +apiVersion: v1 +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: raw-block-pvc +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + resources: + requests: + storage: 64Mi + storageClassName: ceph-xfs diff --git a/tests/integration/data/test_ceph/pv-reader-pod.yaml b/tests/integration/data/test_ceph/pv-reader-pod.yaml new file mode 100755 index 00000000..c87c8691 --- /dev/null +++ b/tests/integration/data/test_ceph/pv-reader-pod.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +apiVersion: v1 +kind: Pod +metadata: + name: pv-reader-test + namespace: default +spec: + restartPolicy: Never + volumes: + - name: pvc-test + persistentVolumeClaim: + claimName: raw-block-pvc + containers: + - name: pv-reader + image: busybox + command: ["/bin/sh", "-c", "cat /pvc/test_file"] + volumeMounts: + - name: pvc-test + mountPath: /pvc diff --git a/tests/integration/data/test_ceph/pv-writer-pod.yaml b/tests/integration/data/test_ceph/pv-writer-pod.yaml new file mode 100755 index 00000000..129849b5 --- /dev/null +++ b/tests/integration/data/test_ceph/pv-writer-pod.yaml @@ -0,0 +1,21 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +apiVersion: v1 +kind: Pod +metadata: + name: pv-writer-test + namespace: default +spec: + restartPolicy: Never + volumes: + - name: pvc-test + persistentVolumeClaim: + claimName: raw-block-pvc + containers: + - name: pv-writer + image: busybox + command: ["/bin/sh", "-c", "echo 'PVC test data.' > /pvc/test_file"] + volumeMounts: + - name: pvc-test + mountPath: /pvc diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index f58037e3..c1cee35a 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -2,6 +2,8 @@ # See LICENSE file for licensing details. """Additions to tools missing from juju library.""" +# pylint: disable=too-many-arguments,too-many-positional-arguments + import ipaddress import json import logging @@ -9,8 +11,9 @@ from typing import List import yaml +from juju import unit from juju.model import Model -from tenacity import retry, stop_after_attempt, wait_fixed +from tenacity import AsyncRetrying, retry, stop_after_attempt, wait_fixed log = logging.getLogger(__name__) @@ -117,3 +120,64 @@ async def ready_nodes(k8s, expected_count): for node, ready in ready_nodes.items(): log.info("Node %s is %s..", node, "ready" if ready else "not ready") assert ready, f"Node not yet ready: {node}." + + +async def wait_pod_phase( + k8s: unit.Unit, + name: str, + phase: str, + namespace: str = "default", + retry_times: int = 30, + retry_delay_s: int = 5, +): + """Wait for the pod to reach the specified phase (e.g. Succeeded). + + Args: + k8s: k8s unit + name: the pod name + phase: expected phase + namespace: pod namespace + retry_times: the number of retries + retry_delay_s: retry interval + + """ + async for attempt in AsyncRetrying( + stop=stop_after_attempt(retry_times), wait=wait_fixed(retry_delay_s) + ): + with attempt: + cmd = " ".join( + [ + "k8s kubectl wait", + f"--namespace {namespace}", + "--for=jsonpath='{.status.phase}'=" + phase, + f"pod/{name}", + "--timeout 1s", + ] + ) + action = await k8s.run(cmd) + result = await action.wait() + assert ( + result.results["return-code"] == 0 + ), f"Failed waiting for pod to reach {phase} phase." + + +async def get_pod_logs( + k8s: unit.Unit, + name: str, + namespace: str = "default", +) -> str: + """Retrieve pod logs. + + Args: + k8s: k8s unit + name: pod name + namespace: pod namespace + + Returns: + the pod logs as string. + """ + cmd = " ".join(["k8s kubectl logs", f"--namespace {namespace}", f"pod/{name}"]) + action = await k8s.run(cmd) + result = await action.wait() + assert result.results["return-code"] == 0, f"Failed to retrieve pod {name} logs." + return result.results["stdout"] diff --git a/tests/integration/test_ceph.py b/tests/integration/test_ceph.py index c526a998..0bfc02d8 100644 --- a/tests/integration/test_ceph.py +++ b/tests/integration/test_ceph.py @@ -6,9 +6,13 @@ # pylint: disable=duplicate-code """Integration tests.""" +from pathlib import Path + import pytest from juju import model, unit +from . import helpers + # This pytest mark configures the test environment to use the Canonical Kubernetes # bundle with ceph, for all the test within this module. pytestmark = [ @@ -17,11 +21,45 @@ ] +def _get_data_file_path(name) -> str: + """Retrieve the full path of the specified test data file.""" + path = Path(__file__).parent / "data" / "test_ceph" / name + return str(path) + + @pytest.mark.abort_on_fail async def test_ceph_sc(kubernetes_cluster: model.Model): - """Test that a ceph storage class is available.""" + """Test that a ceph storage class is available and validate pv attachments.""" k8s: unit.Unit = kubernetes_cluster.applications["k8s"].units[0] event = await k8s.run("k8s kubectl get sc -o=jsonpath='{.items[*].provisioner}'") result = await event.wait() stdout = result.results["stdout"] assert "rbd.csi.ceph.com" in stdout, f"No ceph provisioner found in: {stdout}" + + # Copy pod definitions. + for fname in ["ceph-xfs-pvc.yaml", "pv-writer-pod.yaml", "pv-reader-pod.yaml"]: + await k8s.scp_to(_get_data_file_path(fname), f"/tmp/{fname}") + + # Create "ceph-xfs" PVC. + event = await k8s.run("k8s kubectl apply -f /tmp/ceph-xfs-pvc.yaml") + result = await event.wait() + assert result.results["return-code"] == 0, "Failed to create pvc." + + # Create a pod that writes to the Ceph PV. + event = await k8s.run("k8s kubectl apply -f /tmp/pv-writer-pod.yaml") + result = await event.wait() + assert result.results["return-code"] == 0, "Failed to create writer pod." + + # Wait for the pod to exit successfully. + await helpers.wait_pod_phase(k8s, "pv-writer-test", "Succeeded") + + # Create a pod that reads the PV data and writes it to the log. + event = await k8s.run("k8s kubectl apply -f /tmp/pv-reader-pod.yaml") + result = await event.wait() + assert result.results["return-code"] == 0, "Failed to create reader pod." + + await helpers.wait_pod_phase(k8s, "pv-reader-test", "Succeeded") + + # Check the logged PV data. + logs = await helpers.get_pod_logs(k8s, "pv-reader-test") + assert "PVC test data" in logs diff --git a/tox.ini b/tox.ini index 9241a42b..4ec8b7df 100644 --- a/tox.ini +++ b/tox.ini @@ -89,8 +89,10 @@ description = Run integration tests deps = -r test_requirements.txt commands = pytest -v --tb native \ - --log-cli-level=INFO \ -s {toxinidir}/tests/integration \ + --log-cli-level INFO \ + --log-format "%(asctime)s %(levelname)s %(message)s" \ + --log-date-format "%Y-%m-%d %H:%M:%S" \ --crash-dump=on-failure \ --crash-dump-args='-j snap.k8s.* --as-root' \ {posargs}