diff --git a/charms/worker/charmcraft.yaml b/charms/worker/charmcraft.yaml index 6f158ad0..2d7602ed 100644 --- a/charms/worker/charmcraft.yaml +++ b/charms/worker/charmcraft.yaml @@ -60,7 +60,7 @@ bases: config: options: - labels: + node-labels: default: "" type: string description: | diff --git a/charms/worker/k8s/charmcraft.yaml b/charms/worker/k8s/charmcraft.yaml index 8f062a00..72b2b452 100644 --- a/charms/worker/k8s/charmcraft.yaml +++ b/charms/worker/k8s/charmcraft.yaml @@ -81,7 +81,7 @@ config: Example: e.g.: key1=value1 key2=value2 - containerd_custom_registries: + containerd-custom-registries: type: string default: "[]" description: | @@ -128,39 +128,63 @@ config: "key_file": "'"$(base64 -w 0 < ~/my.custom.key.pem)"'", }]' - datastore: + bootstrap-datastore: default: dqlite type: string description: | The datastore to use in Canonical Kubernetes. This cannot be changed after deployment. Allowed values are "dqlite" and "etcd". If "etcd" is chosen, the charm should be integrated with the etcd charm. - labels: - default: "" - type: string - description: | - Labels can be used to organize and to select subsets of nodes in the - cluster. Declare node labels in key=value format, separated by spaces. - register-with-taints: + bootstrap-node-taints: type: string default: "" description: | Space-separated list of taints to apply to this node at registration time. - This config is only used at deploy time when Kubelet first registers the + This config is only used at bootstrap time when Kubelet first registers the node with Kubernetes. To change node taints after deploy time, use kubectl instead. For more information, see the upstream Kubernetes documentation about taints: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - service-cidr: + bootstrap-pod-cidr: + type: string + default: "10.1.0.0/16" + description: | + Comma-separated CIDR blocks for IP addresses that can be assigned + to pods within the cluster. Can contain at most 2 blocks, one for IPv4 + and one for IPv6. + + After deployment it is not possible to change the size of + the IP range. + + Examples: + - "192.0.2.0/24" + - "2001:db8::/32" + - "192.0.2.0/24,2001:db8::/32" + - "2001:db8::/32,192.0.2.0/24" + bootstrap-service-cidr: type: string default: 10.152.183.0/24 description: | - CIDR to use for Kubernetes services. After deployment it is - only possible to increase the size of the IP range. It is not possible to - change or shrink the address range after deployment. + Comma-separated CIDR blocks for IP addresses that can be assigned + to services within the cluster. Can contain at most 2 blocks, one for IPv4 + and one for IPv6. + + After deployment it is not possible to change the size of + the IP range. + + Examples: + - "192.0.2.0/24" + - "2001:db8::/32" + - "192.0.2.0/24,2001:db8::/32" + - "2001:db8::/32,192.0.2.0/24" + gateway-enabled: + type: boolean + default: false + description: | + Enable/Disable the gateway feature on the cluster. local-storage-enabled: type: boolean default: true @@ -184,16 +208,17 @@ config: "Retain". If set to "Delete", the storage will be deleted when the PersistentVolumeClaim is deleted. If set to "Retain", the storage will be retained when the PersistentVolumeClaim is deleted. - gateway-enabled: - type: boolean - default: false - description: | - Enable/Disable the gateway feature on the cluster. network-enabled: type: boolean default: true description: | Enables or disables the network feature. + node-labels: + default: "" + type: string + description: | + Labels can be used to organize and to select subsets of nodes in the + cluster. Declare node labels in key=value format, separated by spaces. resources: snap-installation: diff --git a/charms/worker/k8s/requirements.txt b/charms/worker/k8s/requirements.txt index 9e532201..3c639667 100644 --- a/charms/worker/k8s/requirements.txt +++ b/charms/worker/k8s/requirements.txt @@ -1,6 +1,6 @@ charm-lib-contextual-status @ git+https://github.com/charmed-kubernetes/charm-lib-contextual-status@255dd4a23defc16dcdac832306e5f460a0f1200c charm-lib-interface-external-cloud-provider @ git+https://github.com/charmed-kubernetes/charm-lib-interface-external-cloud-provider@e1c5fc69e98100a7d43c0ad5a7969bba1ecbcd40 -charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@9b212854e768f13c26cc907bed51444e97e51b50#subdirectory=ops +charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@a14d685237302711113ac651920476437b3b9785#subdirectory=ops charm-lib-reconciler @ git+https://github.com/charmed-kubernetes/charm-lib-reconciler@f818cc30d1a22be43ffdfecf7fbd9c3fd2967502 ops-interface-kube-control @ git+https://github.com/charmed-kubernetes/interface-kube-control.git@main#subdirectory=ops ops.interface_aws @ git+https://github.com/charmed-kubernetes/interface-aws-integration@main#subdirectory=ops diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index 68dd36e2..95fa9950 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -150,7 +150,11 @@ def __init__(self, *args): self.distributor = TokenDistributor(self, self.get_node_name(), self.api_manager) self.collector = TokenCollector(self, self.get_node_name()) self.labeller = LabelMaker( - self, kubeconfig_path=self._internal_kubeconfig, kubectl=KUBECTL_PATH + self, + kubeconfig_path=self._internal_kubeconfig, + kubectl=KUBECTL_PATH, + user_label_key="node-labels", + timeout=15, ) self._stored.set_default(is_dying=False, cluster_name=str()) @@ -324,8 +328,9 @@ def _bootstrap_k8s_snap(self): bootstrap_config = BootstrapConfig.construct() self._configure_datastore(bootstrap_config) bootstrap_config.cluster_config = self._assemble_cluster_config() - bootstrap_config.service_cidr = str(self.config["service-cidr"]) - bootstrap_config.control_plane_taints = str(self.config["register-with-taints"]).split() + bootstrap_config.service_cidr = str(self.config["bootstrap-service-cidr"]) + bootstrap_config.pod_cidr = str(self.config["bootstrap-pod-cidr"]) + bootstrap_config.control_plane_taints = str(self.config["bootstrap-node-taints"]).split() bootstrap_config.extra_sans = [_get_public_address()] bootstrap_config.extra_node_kube_controller_manager_args = { "--cluster-name": self._generate_unique_cluster_name() @@ -354,7 +359,7 @@ def _config_containerd_registries(self): registries, config = [], "" containerd_relation = self.model.get_relation("containerd") if self.is_control_plane: - config = str(self.config["containerd_custom_registries"]) + config = str(self.config["containerd-custom-registries"]) registries = containerd.parse_registries(config) else: registries = containerd.recover(containerd_relation) @@ -416,9 +421,7 @@ def _assemble_cluster_config(self) -> UserFacingClusterConfig: # https://github.com/canonical/k8s-operator/pull/169/files#r1847378214 ) - gateway = GatewayConfig( - enabled=self.config.get("gateway-enabled"), - ) + gateway = GatewayConfig(enabled=self.config.get("gateway-enabled")) network = NetworkConfig( enabled=self.config.get("network-enabled"), @@ -444,7 +447,7 @@ def _configure_datastore(self, config: Union[BootstrapConfig, UpdateClusterConfi The configuration object for the Kubernetes cluster. This object will be modified in-place to include etcd's configuration details. """ - datastore = self.config.get("datastore") + datastore = self.config.get("bootstrap-datastore") if datastore not in SUPPORTED_DATASTORES: log.error( diff --git a/charms/worker/k8s/src/kube_control.py b/charms/worker/k8s/src/kube_control.py index facb5796..02c8ea2f 100644 --- a/charms/worker/k8s/src/kube_control.py +++ b/charms/worker/k8s/src/kube_control.py @@ -25,8 +25,8 @@ def configure(charm: K8sCharmProtocol): status.add(ops.MaintenanceStatus("Configuring Kube Control")) ca_cert, endpoints = "", [f"https://{binding.network.bind_address}:6443"] - labels = str(charm.model.config["labels"]) - taints = str(charm.model.config["register-with-taints"]) + labels = str(charm.model.config["node-labels"]) + taints = str(charm.model.config["bootstrap-node-taints"]) if charm._internal_kubeconfig.exists(): kubeconfig = yaml.safe_load(charm._internal_kubeconfig.read_text()) cluster = kubeconfig["clusters"][0]["cluster"] diff --git a/charms/worker/k8s/tests/unit/test_base.py b/charms/worker/k8s/tests/unit/test_base.py index dc1038ce..a69a6f45 100644 --- a/charms/worker/k8s/tests/unit/test_base.py +++ b/charms/worker/k8s/tests/unit/test_base.py @@ -144,7 +144,7 @@ def test_configure_datastore_bootstrap_config_etcd(harness): harness.disable_hooks() bs_config = BootstrapConfig() - harness.update_config({"datastore": "etcd"}) + harness.update_config({"bootstrap-datastore": "etcd"}) harness.add_relation("etcd", "etcd") with mock.patch.object(harness.charm, "etcd") as mock_etcd: mock_etcd.is_ready = True @@ -182,7 +182,7 @@ def test_configure_datastore_runtime_config_etcd(harness): pytest.skip("Not applicable on workers") harness.disable_hooks() - harness.update_config({"datastore": "etcd"}) + harness.update_config({"bootstrap-datastore": "etcd"}) harness.add_relation("etcd", "etcd") with mock.patch.object(harness.charm, "etcd") as mock_etcd: mock_etcd.is_ready = True @@ -190,6 +190,7 @@ def test_configure_datastore_runtime_config_etcd(harness): mock_etcd.get_connection_string.return_value = "foo:1234,bar:1234" uccr_config = UpdateClusterConfigRequest() harness.charm._configure_datastore(uccr_config) + assert uccr_config.datastore assert uccr_config.datastore.ca_crt == "" assert uccr_config.datastore.client_crt == "" assert uccr_config.datastore.client_key == "" diff --git a/tests/integration/data/test-bundle-ceph.yaml b/tests/integration/data/test-bundle-ceph.yaml index 8b803e9e..37d4c912 100644 --- a/tests/integration/data/test-bundle-ceph.yaml +++ b/tests/integration/data/test-bundle-ceph.yaml @@ -11,6 +11,8 @@ applications: channel: latest/edge constraints: cores=2 mem=8G root-disk=16G num_units: 1 + options: + bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule" k8s-worker: charm: k8s-worker channel: latest/edge diff --git a/tests/integration/data/test-bundle-etcd.yaml b/tests/integration/data/test-bundle-etcd.yaml index 7446af4f..662c984f 100644 --- a/tests/integration/data/test-bundle-etcd.yaml +++ b/tests/integration/data/test-bundle-etcd.yaml @@ -22,7 +22,8 @@ applications: num_units: 1 constraints: cores=2 mem=8G root-disk=16G options: - datastore: etcd + bootstrap-datastore: etcd + bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule" k8s-worker: charm: k8s-worker channel: latest/edge diff --git a/tests/integration/data/test-bundle.yaml b/tests/integration/data/test-bundle.yaml index dcc5710f..76448c31 100644 --- a/tests/integration/data/test-bundle.yaml +++ b/tests/integration/data/test-bundle.yaml @@ -12,6 +12,8 @@ applications: num_units: 3 constraints: cores=2 mem=8G root-disk=16G expose: true + options: + bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule" k8s-worker: charm: k8s-worker channel: latest/edge diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 258c039d..2e6ee6ea 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -54,7 +54,7 @@ async def test_nodes_labelled(request, kubernetes_cluster: model.Model): testname: str = request.node.name k8s: application.Application = kubernetes_cluster.applications["k8s"] worker: application.Application = kubernetes_cluster.applications["k8s-worker"] - label_config = {"labels": f"{testname}="} + label_config = {"node-labels": f"{testname}="} await asyncio.gather(k8s.set_config(label_config), worker.set_config(label_config)) await kubernetes_cluster.wait_for_idle(status="active", timeout=10 * 60)