From 6057f6c75dbd86f65b55d61471daf9658c92c0b9 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Thu, 19 Dec 2024 12:14:29 -0600 Subject: [PATCH] Allow charms to remove node labels via config --- charms/worker/k8s/requirements.txt | 2 +- charms/worker/k8s/src/kube_control.py | 24 ++++++++-- tests/integration/data/test-bundle.yaml | 1 + tests/integration/test_k8s.py | 62 +++++++++++++++++-------- 4 files changed, 64 insertions(+), 25 deletions(-) diff --git a/charms/worker/k8s/requirements.txt b/charms/worker/k8s/requirements.txt index 26779f3a..083fdc63 100644 --- a/charms/worker/k8s/requirements.txt +++ b/charms/worker/k8s/requirements.txt @@ -1,6 +1,6 @@ charm-lib-contextual-status @ git+https://github.com/charmed-kubernetes/charm-lib-contextual-status@255dd4a23defc16dcdac832306e5f460a0f1200c charm-lib-interface-external-cloud-provider @ git+https://github.com/charmed-kubernetes/charm-lib-interface-external-cloud-provider@e1c5fc69e98100a7d43c0ad5a7969bba1ecbcd40 -charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@a14d685237302711113ac651920476437b3b9785#subdirectory=ops +charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@KU-2391/charm-ownership-of-node-role-labels#subdirectory=ops charm-lib-reconciler @ git+https://github.com/charmed-kubernetes/charm-lib-reconciler@f818cc30d1a22be43ffdfecf7fbd9c3fd2967502 ops-interface-kube-control @ git+https://github.com/charmed-kubernetes/interface-kube-control.git@main#subdirectory=ops ops.interface_aws @ git+https://github.com/charmed-kubernetes/interface-aws-integration@main#subdirectory=ops diff --git a/charms/worker/k8s/src/kube_control.py b/charms/worker/k8s/src/kube_control.py index 02c8ea2f..a0fae949 100644 --- a/charms/worker/k8s/src/kube_control.py +++ b/charms/worker/k8s/src/kube_control.py @@ -8,12 +8,31 @@ import charms.contextual_status as status import ops import yaml +from charms.contextual_status import BlockedStatus, on_error from protocols import K8sCharmProtocol # Log messages can be retrieved using juju debug-log log = logging.getLogger(__name__) +@on_error( + BlockedStatus("Invalid config on node-labels or bootstrap-node-taints"), + ValueError, + TypeError, +) +def _share_labels_and_taints(charm: K8sCharmProtocol): + """Share labels and taints with the kube-control interface. + + Args: + charm (K8sCharmProtocol): The charm instance. + """ + labels = str(charm.model.config["node-labels"]) + taints = str(charm.model.config["bootstrap-node-taints"]) + + charm.kube_control.set_labels(labels.split()) + charm.kube_control.set_taints(taints.split()) + + def configure(charm: K8sCharmProtocol): """Configure kube-control for the Kubernetes cluster. @@ -25,8 +44,6 @@ def configure(charm: K8sCharmProtocol): status.add(ops.MaintenanceStatus("Configuring Kube Control")) ca_cert, endpoints = "", [f"https://{binding.network.bind_address}:6443"] - labels = str(charm.model.config["node-labels"]) - taints = str(charm.model.config["bootstrap-node-taints"]) if charm._internal_kubeconfig.exists(): kubeconfig = yaml.safe_load(charm._internal_kubeconfig.read_text()) cluster = kubeconfig["clusters"][0]["cluster"] @@ -52,8 +69,7 @@ def configure(charm: K8sCharmProtocol): charm.kube_control.set_cluster_name(charm.get_cluster_name()) charm.kube_control.set_has_external_cloud_provider(charm.xcp.has_xcp) - charm.kube_control.set_labels(labels.split()) - charm.kube_control.set_taints(taints.split()) + _share_labels_and_taints(charm) for request in charm.kube_control.auth_requests: log.info("Signing kube-control request for '%s 'in '%s'", request.user, request.group) diff --git a/tests/integration/data/test-bundle.yaml b/tests/integration/data/test-bundle.yaml index e35e10db..95d45ffd 100644 --- a/tests/integration/data/test-bundle.yaml +++ b/tests/integration/data/test-bundle.yaml @@ -13,6 +13,7 @@ applications: expose: true options: bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule" + node-labels: "node-role.kubernetes.io/control-plane= k8sd.io/role=control-plane" kube-apiserver-extra-args: "v=3" kube-controller-manager-extra-args: "v=3" kube-proxy-extra-args: "v=3" diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 46b256c7..fb102690 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -11,7 +11,7 @@ import pytest import pytest_asyncio -from juju import application, model +from juju import application, model, unit from tenacity import retry, stop_after_attempt, wait_fixed from .grafana import Grafana @@ -35,35 +35,57 @@ async def test_nodes_ready(kubernetes_cluster: model.Model): await ready_nodes(k8s.units[0], expected_nodes) -async def test_nodes_labelled(request, kubernetes_cluster: model.Model): +@pytest.fixture +async def preserve_charm_config(kubernetes_cluster: model.Model): + """Preserve the charm config changes from a test.""" + k8s: application.Application = kubernetes_cluster.applications["k8s"] + worker: application.Application = kubernetes_cluster.applications["k8s-worker"] + k8s_config, worker_config = await asyncio.gather(k8s.get_config(), worker.get_config()) + yield k8s_config, worker_config + await asyncio.gather(k8s.set_config(k8s_config), worker.set_config(worker_config)) + await kubernetes_cluster.wait_for_idle(status="active", timeout=10 * 60) + + +@pytest.mark.usefixtures("preserve_charm_config") +async def test_node_labels(request, kubernetes_cluster: model.Model): """Test the charms label the nodes appropriately.""" testname: str = request.node.name k8s: application.Application = kubernetes_cluster.applications["k8s"] worker: application.Application = kubernetes_cluster.applications["k8s-worker"] + + # Set a VALID node-label on both k8s and worker label_config = {"node-labels": f"{testname}="} await asyncio.gather(k8s.set_config(label_config), worker.set_config(label_config)) - await kubernetes_cluster.wait_for_idle(status="active", timeout=10 * 60) + await kubernetes_cluster.wait_for_idle(status="active", timeout=5 * 60) - try: - nodes = await get_rsc(k8s.units[0], "nodes") - labelled = [n for n in nodes if testname in n["metadata"]["labels"]] - juju_nodes = [n for n in nodes if "juju-charm" in n["metadata"]["labels"]] - assert len(k8s.units + worker.units) == len( - labelled - ), "Not all nodes labelled with custom-label" - assert len(k8s.units + worker.units) == len( - juju_nodes - ), "Not all nodes labelled as juju-charms" - finally: - await asyncio.gather( - k8s.reset_config(list(label_config)), worker.reset_config(list(label_config)) - ) - - await kubernetes_cluster.wait_for_idle(status="active", timeout=10 * 60) nodes = await get_rsc(k8s.units[0], "nodes") labelled = [n for n in nodes if testname in n["metadata"]["labels"]] juju_nodes = [n for n in nodes if "juju-charm" in n["metadata"]["labels"]] - assert 0 == len(labelled), "Not all nodes labelled with custom-label" + assert len(k8s.units + worker.units) == len( + labelled + ), "Not all nodes labelled with custom-label" + assert len(k8s.units + worker.units) == len( + juju_nodes + ), "Not all nodes labelled as juju-charms" + + # Set an INVALID node-label on both k8s and worker + label_config = {"node-labels": f"{testname}=invalid="} + await asyncio.gather(k8s.set_config(label_config), worker.set_config(label_config)) + await kubernetes_cluster.wait_for_idle(timeout=5 * 60) + leader_idx = await get_leader(k8s) + leader: unit.Unit = k8s.units[leader_idx] + assert leader.workload_status == "blocked", "Leader not blocked" + assert "node-labels" in leader.workload_status_message, "Leader had unexpected warning" + + # Test resetting all label config + await asyncio.gather( + k8s.reset_config(list(label_config)), worker.reset_config(list(label_config)) + ) + await kubernetes_cluster.wait_for_idle(status="active", timeout=5 * 60) + nodes = await get_rsc(k8s.units[0], "nodes") + labelled = [n for n in nodes if testname in n["metadata"]["labels"]] + juju_nodes = [n for n in nodes if "juju-charm" in n["metadata"]["labels"]] + assert 0 == len(labelled), "Not all nodes labelled without custom-label" @pytest.mark.abort_on_fail