diff --git a/charms/worker/charmcraft.yaml b/charms/worker/charmcraft.yaml index 6f158ad0..2d7602ed 100644 --- a/charms/worker/charmcraft.yaml +++ b/charms/worker/charmcraft.yaml @@ -60,7 +60,7 @@ bases: config: options: - labels: + node-labels: default: "" type: string description: | diff --git a/charms/worker/k8s/charmcraft.yaml b/charms/worker/k8s/charmcraft.yaml index 19f26f4c..09ec05cf 100644 --- a/charms/worker/k8s/charmcraft.yaml +++ b/charms/worker/k8s/charmcraft.yaml @@ -81,7 +81,59 @@ config: Example: e.g.: key1=value1 key2=value2 - containerd_custom_registries: + bootstrap-datastore: + default: dqlite + type: string + description: | + The datastore to use in Canonical Kubernetes. This cannot be changed + after deployment. Allowed values are "dqlite" and "etcd". If "etcd" is + chosen, the charm should be integrated with the etcd charm. + bootstrap-node-taints: + type: string + default: "" + description: | + Space-separated list of taints to apply to this node at registration time. + + This config is only used at bootstrap time when Kubelet first registers the + node with Kubernetes. To change node taints after deploy time, use kubectl + instead. + + For more information, see the upstream Kubernetes documentation about + taints: + https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + bootstrap-pod-cidr: + type: string + default: "10.1.0.0/16" + description: | + Comma-separated CIDR blocks for IP addresses that can be assigned + to pods within the cluster. Can contain at most 2 blocks, one for IPv4 + and one for IPv6. + + After deployment it is not possible to change the size of + the IP range. + + Examples: + - "192.0.2.0/24" + - "2001:db8::/32" + - "192.0.2.0/24,2001:db8::/32" + - "2001:db8::/32,192.0.2.0/24" + bootstrap-service-cidr: + type: string + default: 10.152.183.0/24 + description: | + Comma-separated CIDR blocks for IP addresses that can be assigned + to services within the cluster. Can contain at most 2 blocks, one for IPv4 + and one for IPv6. + + After deployment it is not possible to change the size of + the IP range. + + Examples: + - "192.0.2.0/24" + - "2001:db8::/32" + - "192.0.2.0/24,2001:db8::/32" + - "2001:db8::/32,192.0.2.0/24" + containerd-custom-registries: type: string default: "[]" description: | @@ -127,40 +179,85 @@ config: "cert_file": "'"$(base64 -w 0 < ~/my.custom.cert.pem)"'", "key_file": "'"$(base64 -w 0 < ~/my.custom.key.pem)"'", }]' - - datastore: - default: dqlite + dns-enabled: + type: boolean + default: true + description: | + Enable/Disable the DNS feature on the cluster. + dns-cluster-domain: type: string + default: "cluster.local" description: | - The datastore to use in Canonical Kubernetes. This cannot be changed - after deployment. Allowed values are "dqlite" and "etcd". If "etcd" is - chosen, the charm should be integrated with the etcd charm. - labels: + Sets the local domain of the cluster + dns-service-ip: + type: string default: "" + description: | + Sets the IP address of the dns service. If omitted defaults to the IP address + of the Kubernetes service created by the feature. + + Can be used to point to an external dns server when feature is disabled. + dns-upstream-nameservers: type: string + default: "" description: | - Labels can be used to organize and to select subsets of nodes in the - cluster. Declare node labels in key=value format, separated by spaces. - register-with-taints: + Space-separated list of upstream nameservers used to forward queries for out-of-cluster + endpoints. + + If omitted defaults to `/etc/resolv.conf` and uses the nameservers on each node. + gateway-enabled: + type: boolean + default: false + description: | + Enable/Disable the gateway feature on the cluster. + load-balancer-enabled: + type: boolean + default: false + description: | + Enable/Disable the load balancer feature on the cluster. + load-balancer-cidrs: type: string default: "" description: | - Space-separated list of taints to apply to this node at registration time. - - This config is only used at deploy time when Kubelet first registers the - node with Kubernetes. To change node taints after deploy time, use kubectl - instead. - - For more information, see the upstream Kubernetes documentation about - taints: - https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - service-cidr: + Space-separated list of CIDRs to use for the load balancer. This is + only used if load-balancer-enabled is set to true. + load-balancer-l2-mode: + type: boolean + default: false + description: | + Enable/Disable L2 mode for the load balancer. This is only used if + load-balancer-enabled is set to true. + load-balancer-l2-interfaces: type: string - default: 10.152.183.0/24 + default: "" description: | - CIDR to use for Kubernetes services. After deployment it is - only possible to increase the size of the IP range. It is not possible to - change or shrink the address range after deployment. + Space-separated list of interfaces to use for the load balancer. This + is only used if load-balancer-l2-mode is set to true. if unset, all + interfaces will be used. + load-balancer-bgp-mode: + type: boolean + default: false + description: | + Enable/Disable BGP mode for the load balancer. This is only used if + load-balancer-enabled is set to true. + load-balancer-bgp-local-asn: + type: int + default: 64512 + description: | + Local ASN for the load balancer. This is only used if load-balancer-bgp-mode + is set to true. + load-balancer-bgp-peer-address: + type: string + default: "" + description: | + Address of the BGP peer for the load balancer. This is only used if + load-balancer-bgp-mode is set to true. + load-balancer-bgp-peer-port: + type: int + default: 179 + description: | + Port of the BGP peer for the load balancer. This is only used if + load-balancer-bgp-mode is set to true. local-storage-enabled: type: boolean default: true @@ -184,11 +281,22 @@ config: "Retain". If set to "Delete", the storage will be deleted when the PersistentVolumeClaim is deleted. If set to "Retain", the storage will be retained when the PersistentVolumeClaim is deleted. - gateway-enabled: + metrics-server-enabled: type: boolean - default: false + default: true description: | - Enable/Disable the gateway feature on the cluster. + Enable/Disable the metrics-server feature on the cluster. + network-enabled: + type: boolean + default: true + description: | + Enables or disables the network feature. + node-labels: + default: "" + type: string + description: | + Labels can be used to organize and to select subsets of nodes in the + cluster. Declare node labels in key=value format, separated by spaces. resources: snap-installation: diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index 6b0cf534..12234de1 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -230,9 +230,9 @@ class LoadBalancerConfig(BaseModel, allow_population_by_field_name=True): Attributes: enabled: Optional flag which represents the status of LoadBalancer. cidrs: List of CIDR blocks for the load balancer. - l2_enabled: Optional flag to enable or disable layer 2 functionality. + l2_mode: Optional flag to enable or disable layer 2 mode. l2_interfaces: List of layer 2 interfaces for the load balancer. - bgp_enabled: Optional flag to enable or disable BGP. + bgp_mode: Optional flag to enable or disable BGP. bgp_local_asn: The local ASN for BGP configuration. bgp_peer_address: The peer address for BGP configuration. bgp_peer_asn: The peer ASN for BGP configuration. @@ -241,9 +241,9 @@ class LoadBalancerConfig(BaseModel, allow_population_by_field_name=True): enabled: Optional[bool] = Field(default=None) cidrs: Optional[List[str]] = Field(default=None) - l2_enabled: Optional[bool] = Field(default=None, alias="l2-enabled") + l2_mode: Optional[bool] = Field(default=None, alias="l2-mode") l2_interfaces: Optional[List[str]] = Field(default=None, alias="l2-interfaces") - bgp_enabled: Optional[bool] = Field(default=None, alias="bgp-enabled") + bgp_mode: Optional[bool] = Field(default=None, alias="bgp-mode") bgp_local_asn: Optional[int] = Field(default=None, alias="bgp-local-asn") bgp_peer_address: Optional[str] = Field(default=None, alias="bgp-peer-address") bgp_peer_asn: Optional[int] = Field(default=None, alias="bgp-peer-asn") diff --git a/charms/worker/k8s/requirements.txt b/charms/worker/k8s/requirements.txt index 9e532201..3c639667 100644 --- a/charms/worker/k8s/requirements.txt +++ b/charms/worker/k8s/requirements.txt @@ -1,6 +1,6 @@ charm-lib-contextual-status @ git+https://github.com/charmed-kubernetes/charm-lib-contextual-status@255dd4a23defc16dcdac832306e5f460a0f1200c charm-lib-interface-external-cloud-provider @ git+https://github.com/charmed-kubernetes/charm-lib-interface-external-cloud-provider@e1c5fc69e98100a7d43c0ad5a7969bba1ecbcd40 -charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@9b212854e768f13c26cc907bed51444e97e51b50#subdirectory=ops +charm-lib-node-base @ git+https://github.com/charmed-kubernetes/layer-kubernetes-node-base@a14d685237302711113ac651920476437b3b9785#subdirectory=ops charm-lib-reconciler @ git+https://github.com/charmed-kubernetes/charm-lib-reconciler@f818cc30d1a22be43ffdfecf7fbd9c3fd2967502 ops-interface-kube-control @ git+https://github.com/charmed-kubernetes/interface-kube-control.git@main#subdirectory=ops ops.interface_aws @ git+https://github.com/charmed-kubernetes/interface-aws-integration@main#subdirectory=ops diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index 722106b3..e57809bc 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -45,7 +45,9 @@ JoinClusterRequest, K8sdAPIManager, K8sdConnectionError, + LoadBalancerConfig, LocalStorageConfig, + MetricsServerConfig, NetworkConfig, UnixSocketConnectionFactory, UpdateClusterConfigRequest, @@ -150,7 +152,11 @@ def __init__(self, *args): self.distributor = TokenDistributor(self, self.get_node_name(), self.api_manager) self.collector = TokenCollector(self, self.get_node_name()) self.labeller = LabelMaker( - self, kubeconfig_path=self._internal_kubeconfig, kubectl=KUBECTL_PATH + self, + kubeconfig_path=self._internal_kubeconfig, + kubectl=KUBECTL_PATH, + user_label_key="node-labels", + timeout=15, ) self._stored.set_default(is_dying=False, cluster_name=str()) @@ -187,7 +193,6 @@ def _k8s_info(self, event: ops.EventBase): @status.on_error( ops.WaitingStatus("Installing COS requirements"), subprocess.CalledProcessError, - AssertionError, ) def _apply_cos_requirements(self): """Apply COS requirements for integration. @@ -311,7 +316,7 @@ def _check_k8sd_ready(self): @on_error( ops.WaitingStatus("Waiting to bootstrap k8s snap"), - AssertionError, + ReconcilerError, InvalidResponseError, K8sdConnectionError, ) @@ -324,8 +329,9 @@ def _bootstrap_k8s_snap(self): bootstrap_config = BootstrapConfig.construct() self._configure_datastore(bootstrap_config) bootstrap_config.cluster_config = self._assemble_cluster_config() - bootstrap_config.service_cidr = str(self.config["service-cidr"]) - bootstrap_config.control_plane_taints = str(self.config["register-with-taints"]).split() + bootstrap_config.service_cidr = str(self.config["bootstrap-service-cidr"]) + bootstrap_config.pod_cidr = str(self.config["bootstrap-pod-cidr"]) + bootstrap_config.control_plane_taints = str(self.config["bootstrap-node-taints"]).split() bootstrap_config.extra_sans = [_get_public_address()] bootstrap_config.extra_node_kube_controller_manager_args = { "--cluster-name": self._generate_unique_cluster_name() @@ -354,7 +360,7 @@ def _config_containerd_registries(self): registries, config = [], "" containerd_relation = self.model.get_relation("containerd") if self.is_control_plane: - config = str(self.config["containerd_custom_registries"]) + config = str(self.config["containerd-custom-registries"]) registries = containerd.parse_registries(config) else: registries = containerd.recover(containerd_relation) @@ -382,7 +388,7 @@ def _get_valid_annotations(self) -> Optional[dict]: dict: The parsed annotations if valid, otherwise None. Raises: - AssertionError: If any annotation is invalid. + ReconcilerError: If any annotation is invalid. """ raw_annotations = self.config.get("annotations") if not raw_annotations: @@ -393,9 +399,10 @@ def _get_valid_annotations(self) -> Optional[dict]: annotations = {} try: for key, value in [pair.split("=", 1) for pair in raw_annotations.split()]: - assert key and value, "Invalid Annotation" # nosec + if not key or not value: + raise ReconcilerError("Invalid Annotation") annotations[key] = value - except AssertionError: + except ReconcilerError: log.exception("Invalid annotations: %s", raw_annotations) status.add(ops.BlockedStatus("Invalid Annotations")) raise @@ -416,8 +423,34 @@ def _assemble_cluster_config(self) -> UserFacingClusterConfig: # https://github.com/canonical/k8s-operator/pull/169/files#r1847378214 ) - gateway = GatewayConfig( - enabled=self.config.get("gateway-enabled"), + dns_config = DNSConfig( + enabled=self.config.get("dns-enabled"), + ) + if cfg := self.config.get("dns-cluster-domain"): + dns_config.cluster_domain = str(cfg) + if cfg := self.config.get("dns-service-ip"): + dns_config.service_ip = str(cfg) + if cfg := self.config.get("dns-upstream-nameservers"): + dns_config.upstream_nameservers = str(cfg).split() + + gateway = GatewayConfig(enabled=self.config.get("gateway-enabled")) + + network = NetworkConfig( + enabled=self.config.get("network-enabled"), + ) + + metrics_server = MetricsServerConfig(enabled=self.config.get("metrics-server-enabled")) + + load_balancer = LoadBalancerConfig( + enabled=self.config.get("load-balancer-enabled"), + cidrs=str(self.config.get("load-balancer-cidrs")).split(), + l2_mode=self.config.get("load-balancer-l2-mode"), + l2_interfaces=str(self.config.get("load-balancer-l2-interfaces")).split(), + bgp_mode=self.config.get("load-balancer-bgp-mode"), + bgp_local_asn=self.config.get("load-balancer-bgp-local-asn"), + bgp_peer_address=self.config.get("load-balancer-bgp-peer-address"), + bgp_peer_asn=self.config.get("load-balancer-bgp-peer-asn"), + bgp_peer_port=self.config.get("load-balancer-bgp-peer-port"), ) cloud_provider = None @@ -425,10 +458,14 @@ def _assemble_cluster_config(self) -> UserFacingClusterConfig: cloud_provider = "external" return UserFacingClusterConfig( - local_storage=local_storage, - gateway=gateway, annotations=self._get_valid_annotations(), cloud_provider=cloud_provider, + dns_config=dns_config, + gateway=gateway, + local_storage=local_storage, + load_balancer=load_balancer, + metrics_server=metrics_server, + network=network, ) def _configure_datastore(self, config: Union[BootstrapConfig, UpdateClusterConfigRequest]): @@ -439,7 +476,7 @@ def _configure_datastore(self, config: Union[BootstrapConfig, UpdateClusterConfi The configuration object for the Kubernetes cluster. This object will be modified in-place to include etcd's configuration details. """ - datastore = self.config.get("datastore") + datastore = self.config.get("bootstrap-datastore") if datastore not in SUPPORTED_DATASTORES: log.error( @@ -448,14 +485,18 @@ def _configure_datastore(self, config: Union[BootstrapConfig, UpdateClusterConfi ", ".join(SUPPORTED_DATASTORES), ) status.add(ops.BlockedStatus(f"Invalid datastore: {datastore}")) - assert datastore in SUPPORTED_DATASTORES # nosec + if datastore not in SUPPORTED_DATASTORES: + raise ReconcilerError(f"Invalid datastore: {datastore}") if datastore == "etcd": log.info("Using etcd as external datastore") etcd_relation = self.model.get_relation("etcd") - assert etcd_relation, "Missing etcd relation" # nosec - assert self.etcd.is_ready, "etcd is not ready" # nosec + if not etcd_relation: + raise ReconcilerError("Missing etcd relation") + + if not self.etcd.is_ready: + raise ReconcilerError("etcd is not ready") etcd_config = self.etcd.get_client_credentials() if isinstance(config, BootstrapConfig): @@ -551,28 +592,9 @@ def _create_cos_tokens(self): token_type=ClusterTokenType.WORKER, ) - @on_error( - WaitingStatus("Waiting to enable features"), - InvalidResponseError, - K8sdConnectionError, - ) - def _enable_functionalities(self): - """Enable necessary components for the Kubernetes cluster.""" - status.add(ops.MaintenanceStatus("Updating K8s features")) - log.info("Enabling K8s features") - dns_config = DNSConfig(enabled=True) - network_config = NetworkConfig(enabled=True) - local_storage_config = LocalStorageConfig(enabled=True) - user_cluster_config = UserFacingClusterConfig( - dns=dns_config, network=network_config, local_storage=local_storage_config - ) - update_request = UpdateClusterConfigRequest(config=user_cluster_config) - - self.api_manager.update_cluster_config(update_request) - @on_error( WaitingStatus("Ensure that the cluster configuration is up-to-date"), - AssertionError, + ReconcilerError, InvalidResponseError, K8sdConnectionError, ) @@ -610,7 +632,7 @@ def _get_scrape_jobs(self): return self.cos.get_metrics_endpoints( self.get_node_name(), token, self.is_control_plane ) - except AssertionError: + except ReconcilerError: log.exception("Failed to get COS token.") return [] @@ -682,7 +704,7 @@ def _get_proxy_env(self) -> Dict[str, str]: @on_error( WaitingStatus("Waiting for Cluster token"), - AssertionError, + ReconcilerError, InvalidResponseError, K8sdConnectionError, ) @@ -768,12 +790,11 @@ def _reconcile(self, event: ops.EventBase): if self.lead_control_plane: self._k8s_info(event) self._bootstrap_k8s_snap() - self._enable_functionalities() + self._ensure_cluster_config() self._create_cluster_tokens() self._create_cos_tokens() self._apply_cos_requirements() self._revoke_cluster_tokens(event) - self._ensure_cluster_config() self._announce_kubernetes_version() self._join_cluster(event) self._config_containerd_registries() diff --git a/charms/worker/k8s/src/kube_control.py b/charms/worker/k8s/src/kube_control.py index facb5796..02c8ea2f 100644 --- a/charms/worker/k8s/src/kube_control.py +++ b/charms/worker/k8s/src/kube_control.py @@ -25,8 +25,8 @@ def configure(charm: K8sCharmProtocol): status.add(ops.MaintenanceStatus("Configuring Kube Control")) ca_cert, endpoints = "", [f"https://{binding.network.bind_address}:6443"] - labels = str(charm.model.config["labels"]) - taints = str(charm.model.config["register-with-taints"]) + labels = str(charm.model.config["node-labels"]) + taints = str(charm.model.config["bootstrap-node-taints"]) if charm._internal_kubeconfig.exists(): kubeconfig = yaml.safe_load(charm._internal_kubeconfig.read_text()) cluster = kubeconfig["clusters"][0]["cluster"] diff --git a/charms/worker/k8s/src/token_distributor.py b/charms/worker/k8s/src/token_distributor.py index c3d65c41..d51d91bb 100644 --- a/charms/worker/k8s/src/token_distributor.py +++ b/charms/worker/k8s/src/token_distributor.py @@ -11,6 +11,7 @@ import charms.contextual_status as status import ops +from charms.contextual_status import ReconcilerError from charms.k8s.v0.k8sd_api_manager import ( ErrorCodes, InvalidResponseError, @@ -209,6 +210,9 @@ def cluster_name(self, relation: ops.Relation, local: bool) -> str: Returns: the recovered cluster name from existing relations + + Raises: + ReconcilerError: If fails to find 1 relation-name:cluster-name. """ cluster_name: Optional[str] = "" if not local: @@ -218,7 +222,8 @@ def cluster_name(self, relation: ops.Relation, local: bool) -> str: if value := relation.data[unit].get("cluster-name"): values |= {value} if values: - assert len(values) == 1, f"Failed to find 1 {relation.name}:cluster-name" # nosec + if len(values) != 1: + raise ReconcilerError(f"Failed to find 1 {relation.name}:cluster-name") (cluster_name,) = values elif not (cluster_name := relation.data[self.charm.unit].get("joined")): # joined_cluster_name @@ -235,6 +240,12 @@ def recover_token(self, relation: ops.Relation) -> Generator[str, None, None]: Yields: str: extracted token content + + Raises: + ReconcilerError: + - If fails to find 1 relation-name:secret-id. + - If relation-name:secret-key is not valid. + - If relation-name:token is not valid. """ self.request(relation) @@ -246,14 +257,17 @@ def recover_token(self, relation: ops.Relation) -> Generator[str, None, None]: if (secret_id := relation.data[unit].get(secret_key)) } - assert len(secret_ids) == 1, f"Failed to find 1 {relation.name}:{secret_key}" # nosec + if len(secret_ids) != 1: + raise ReconcilerError(f"Failed to find 1 {relation.name}:{secret_key}") (secret_id,) = secret_ids - assert secret_id, f"{relation.name}:{secret_key} is not valid" # nosec + if not secret_id: + raise ReconcilerError(f"{relation.name}:{secret_key} is not valid") secret = self.charm.model.get_secret(id=secret_id) # Get the content from the secret content = secret.get_content(refresh=True) - assert content["token"], f"{relation.name}:token not valid" # nosec + if not content.get("token"): + raise ReconcilerError(f"{relation.name}:token not valid") yield content["token"] # signal that the relation is joined, the token is used @@ -343,7 +357,7 @@ def update_node(self, relation: ops.Relation, unit: ops.Unit, state: str): """ relation.data[self.charm.app][unit.name] = state - def allocate_tokens( + def allocate_tokens( # noqa: C901 self, relation: ops.Relation, token_strategy: TokenStrategy, @@ -356,16 +370,23 @@ def allocate_tokens( token_strategy (TokenStrategy): The strategy of token creation. token_type (ClusterTokenType): The type of cluster token. Defaults to ClusterTokenType.NONE. + + Raises: + ReconcilerError: + - If token_strategy is valid. + - If remote application doesn't exist on relation. """ units = relation.units if self.charm.app == relation.app: # include self in peer relations units |= {self.charm.unit} - assert relation.app, f"Remote application doesn't exist on {relation.name}" # nosec + if not relation.app: + raise ReconcilerError(f"Remote application doesn't exist on {relation.name}") # Select the appropriate token creation strategy tokenizer = self.token_strategies.get(token_strategy) - assert tokenizer, f"Invalid token_strategy: {token_strategy}" # nosec + if not tokenizer: + raise ReconcilerError(f"Invalid token_strategy: {token_strategy}") log.info("Allocating %s %s tokens", token_type.name.title(), token_strategy.name.title()) status.add( diff --git a/charms/worker/k8s/tests/unit/test_base.py b/charms/worker/k8s/tests/unit/test_base.py index dc1038ce..6a3f419a 100644 --- a/charms/worker/k8s/tests/unit/test_base.py +++ b/charms/worker/k8s/tests/unit/test_base.py @@ -58,7 +58,6 @@ def mock_reconciler_handlers(harness): if harness.charm.is_control_plane: handler_names |= { "_bootstrap_k8s_snap", - "_enable_functionalities", "_create_cluster_tokens", "_create_cos_tokens", "_apply_cos_requirements", @@ -144,7 +143,7 @@ def test_configure_datastore_bootstrap_config_etcd(harness): harness.disable_hooks() bs_config = BootstrapConfig() - harness.update_config({"datastore": "etcd"}) + harness.update_config({"bootstrap-datastore": "etcd"}) harness.add_relation("etcd", "etcd") with mock.patch.object(harness.charm, "etcd") as mock_etcd: mock_etcd.is_ready = True @@ -182,7 +181,7 @@ def test_configure_datastore_runtime_config_etcd(harness): pytest.skip("Not applicable on workers") harness.disable_hooks() - harness.update_config({"datastore": "etcd"}) + harness.update_config({"bootstrap-datastore": "etcd"}) harness.add_relation("etcd", "etcd") with mock.patch.object(harness.charm, "etcd") as mock_etcd: mock_etcd.is_ready = True @@ -190,6 +189,7 @@ def test_configure_datastore_runtime_config_etcd(harness): mock_etcd.get_connection_string.return_value = "foo:1234,bar:1234" uccr_config = UpdateClusterConfigRequest() harness.charm._configure_datastore(uccr_config) + assert uccr_config.datastore assert uccr_config.datastore.ca_crt == "" assert uccr_config.datastore.client_crt == "" assert uccr_config.datastore.client_key == "" diff --git a/charms/worker/k8s/tests/unit/test_config_options.py b/charms/worker/k8s/tests/unit/test_config_options.py new file mode 100644 index 00000000..13bb4f49 --- /dev/null +++ b/charms/worker/k8s/tests/unit/test_config_options.py @@ -0,0 +1,52 @@ +# Copyright 2024 Canonical Ltd. +# See LICENSE file for licensing details. + +# Learn more about testing at: https://juju.is/docs/sdk/testing + +# pylint: disable=duplicate-code,missing-function-docstring +"""Unit tests.""" + + +from pathlib import Path + +import ops +import ops.testing +import pytest +from charm import K8sCharm + + +@pytest.fixture(params=["worker", "control-plane"]) +def harness(request): + """Craft a ops test harness. + + Args: + request: pytest request object + """ + meta = Path(__file__).parent / "../../charmcraft.yaml" + if request.param == "worker": + meta = Path(__file__).parent / "../../../charmcraft.yaml" + harness = ops.testing.Harness(K8sCharm, meta=meta.read_text()) + harness.begin() + harness.charm.is_worker = request.param == "worker" + yield harness + harness.cleanup() + + +def test_configure_network_options(harness): + """Test configuring the network options. + + Args: + harness: the harness under test + """ + if harness.charm.is_worker: + pytest.skip("Not applicable on workers") + + harness.disable_hooks() + + harness.update_config({"network-enabled": False}) + ufcg = harness.charm._assemble_cluster_config() + assert not ufcg.network.enabled, "Network should be disabled" + + harness.update_config({"network-enabled": True}) + ufcg = harness.charm._assemble_cluster_config() + assert ufcg.network.enabled, "Network should be enabled" diff --git a/pyproject.toml b/pyproject.toml index fdf9c317..bffeb7fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,4 +86,4 @@ max-complexity = 10 skip = "build,lib,venv,icon.svg,.tox,.git,.mypy_cache,.ruff_cache,.coverage" [tool.pyright] -extraPaths = ["./charms/worker/k8s/lib","./charms/worker/k8s/src"] +extraPaths = ["./charms/worker/k8s/lib", "./charms/worker/k8s/src"] diff --git a/tests/integration/data/test-bundle-ceph.yaml b/tests/integration/data/test-bundle-ceph.yaml index 8b803e9e..37d4c912 100644 --- a/tests/integration/data/test-bundle-ceph.yaml +++ b/tests/integration/data/test-bundle-ceph.yaml @@ -11,6 +11,8 @@ applications: channel: latest/edge constraints: cores=2 mem=8G root-disk=16G num_units: 1 + options: + bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule" k8s-worker: charm: k8s-worker channel: latest/edge diff --git a/tests/integration/data/test-bundle-etcd.yaml b/tests/integration/data/test-bundle-etcd.yaml index 7446af4f..662c984f 100644 --- a/tests/integration/data/test-bundle-etcd.yaml +++ b/tests/integration/data/test-bundle-etcd.yaml @@ -22,7 +22,8 @@ applications: num_units: 1 constraints: cores=2 mem=8G root-disk=16G options: - datastore: etcd + bootstrap-datastore: etcd + bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule" k8s-worker: charm: k8s-worker channel: latest/edge diff --git a/tests/integration/data/test-bundle.yaml b/tests/integration/data/test-bundle.yaml index dcc5710f..76448c31 100644 --- a/tests/integration/data/test-bundle.yaml +++ b/tests/integration/data/test-bundle.yaml @@ -12,6 +12,8 @@ applications: num_units: 3 constraints: cores=2 mem=8G root-disk=16G expose: true + options: + bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule" k8s-worker: charm: k8s-worker channel: latest/edge diff --git a/tests/integration/data/test_ceph/pv-reader-pod.yaml b/tests/integration/data/test_ceph/pv-reader-pod.yaml index c87c8691..9f75c946 100755 --- a/tests/integration/data/test_ceph/pv-reader-pod.yaml +++ b/tests/integration/data/test_ceph/pv-reader-pod.yaml @@ -14,7 +14,7 @@ spec: claimName: raw-block-pvc containers: - name: pv-reader - image: busybox + image: rocks.canonical.com/cdk/busybox:1.36 command: ["/bin/sh", "-c", "cat /pvc/test_file"] volumeMounts: - name: pvc-test diff --git a/tests/integration/data/test_ceph/pv-writer-pod.yaml b/tests/integration/data/test_ceph/pv-writer-pod.yaml index 129849b5..7b659add 100755 --- a/tests/integration/data/test_ceph/pv-writer-pod.yaml +++ b/tests/integration/data/test_ceph/pv-writer-pod.yaml @@ -14,7 +14,7 @@ spec: claimName: raw-block-pvc containers: - name: pv-writer - image: busybox + image: rocks.canonical.com/cdk/busybox:1.36 command: ["/bin/sh", "-c", "echo 'PVC test data.' > /pvc/test_file"] volumeMounts: - name: pvc-test diff --git a/tests/integration/helpers.py b/tests/integration/helpers.py index c1cee35a..dae1b2ba 100644 --- a/tests/integration/helpers.py +++ b/tests/integration/helpers.py @@ -7,13 +7,14 @@ import ipaddress import json import logging +import shlex from pathlib import Path from typing import List import yaml from juju import unit from juju.model import Model -from tenacity import AsyncRetrying, retry, stop_after_attempt, wait_fixed +from tenacity import AsyncRetrying, before_sleep_log, retry, stop_after_attempt, wait_fixed log = logging.getLogger(__name__) @@ -125,10 +126,10 @@ async def ready_nodes(k8s, expected_count): async def wait_pod_phase( k8s: unit.Unit, name: str, - phase: str, + *phase: str, namespace: str = "default", retry_times: int = 30, - retry_delay_s: int = 5, + retry_delay_s: int = 15, ): """Wait for the pod to reach the specified phase (e.g. Succeeded). @@ -142,23 +143,34 @@ async def wait_pod_phase( """ async for attempt in AsyncRetrying( - stop=stop_after_attempt(retry_times), wait=wait_fixed(retry_delay_s) + stop=stop_after_attempt(retry_times), + wait=wait_fixed(retry_delay_s), + before_sleep=before_sleep_log(log, logging.WARNING), ): with attempt: - cmd = " ".join( + cmd = shlex.join( [ - "k8s kubectl wait", - f"--namespace {namespace}", - "--for=jsonpath='{.status.phase}'=" + phase, + "k8s", + "kubectl", + "get", + "--namespace", + namespace, + "-o", + "jsonpath={.status.phase}", f"pod/{name}", - "--timeout 1s", ] ) action = await k8s.run(cmd) result = await action.wait() - assert ( - result.results["return-code"] == 0 - ), f"Failed waiting for pod to reach {phase} phase." + stdout, stderr = ( + result.results.get(field, "").strip() for field in ["stdout", "stderr"] + ) + assert result.results["return-code"] == 0, ( + f"\nPod hasn't reached phase: {phase}\n" + f"\tstdout: '{stdout}'\n" + f"\tstderr: '{stderr}'" + ) + assert stdout in phase, f"Pod {name} not yet in phase {phase} ({stdout})" async def get_pod_logs( diff --git a/tests/integration/test_ceph.py b/tests/integration/test_ceph.py index 0bfc02d8..61bcb39f 100644 --- a/tests/integration/test_ceph.py +++ b/tests/integration/test_ceph.py @@ -37,29 +37,36 @@ async def test_ceph_sc(kubernetes_cluster: model.Model): assert "rbd.csi.ceph.com" in stdout, f"No ceph provisioner found in: {stdout}" # Copy pod definitions. - for fname in ["ceph-xfs-pvc.yaml", "pv-writer-pod.yaml", "pv-reader-pod.yaml"]: + definitions = ["ceph-xfs-pvc.yaml", "pv-writer-pod.yaml", "pv-reader-pod.yaml"] + for fname in definitions: await k8s.scp_to(_get_data_file_path(fname), f"/tmp/{fname}") - # Create "ceph-xfs" PVC. - event = await k8s.run("k8s kubectl apply -f /tmp/ceph-xfs-pvc.yaml") - result = await event.wait() - assert result.results["return-code"] == 0, "Failed to create pvc." - - # Create a pod that writes to the Ceph PV. - event = await k8s.run("k8s kubectl apply -f /tmp/pv-writer-pod.yaml") - result = await event.wait() - assert result.results["return-code"] == 0, "Failed to create writer pod." - - # Wait for the pod to exit successfully. - await helpers.wait_pod_phase(k8s, "pv-writer-test", "Succeeded") - - # Create a pod that reads the PV data and writes it to the log. - event = await k8s.run("k8s kubectl apply -f /tmp/pv-reader-pod.yaml") - result = await event.wait() - assert result.results["return-code"] == 0, "Failed to create reader pod." - - await helpers.wait_pod_phase(k8s, "pv-reader-test", "Succeeded") - - # Check the logged PV data. - logs = await helpers.get_pod_logs(k8s, "pv-reader-test") - assert "PVC test data" in logs + try: + # Create "ceph-xfs" PVC. + event = await k8s.run("k8s kubectl apply -f /tmp/ceph-xfs-pvc.yaml") + result = await event.wait() + assert result.results["return-code"] == 0, "Failed to create pvc." + + # Create a pod that writes to the Ceph PV. + event = await k8s.run("k8s kubectl apply -f /tmp/pv-writer-pod.yaml") + result = await event.wait() + assert result.results["return-code"] == 0, "Failed to create writer pod." + + # Wait for the pod to exit successfully. + await helpers.wait_pod_phase(k8s, "pv-writer-test", "Succeeded") + + # Create a pod that reads the PV data and writes it to the log. + event = await k8s.run("k8s kubectl apply -f /tmp/pv-reader-pod.yaml") + result = await event.wait() + assert result.results["return-code"] == 0, "Failed to create reader pod." + + await helpers.wait_pod_phase(k8s, "pv-reader-test", "Succeeded") + + # Check the logged PV data. + logs = await helpers.get_pod_logs(k8s, "pv-reader-test") + assert "PVC test data" in logs + finally: + # Cleanup + for fname in reversed(definitions): + event = await k8s.run(f"k8s kubectl delete -f /tmp/{fname}") + result = await event.wait() diff --git a/tests/integration/test_k8s.py b/tests/integration/test_k8s.py index 258c039d..2e6ee6ea 100644 --- a/tests/integration/test_k8s.py +++ b/tests/integration/test_k8s.py @@ -54,7 +54,7 @@ async def test_nodes_labelled(request, kubernetes_cluster: model.Model): testname: str = request.node.name k8s: application.Application = kubernetes_cluster.applications["k8s"] worker: application.Application = kubernetes_cluster.applications["k8s-worker"] - label_config = {"labels": f"{testname}="} + label_config = {"node-labels": f"{testname}="} await asyncio.gather(k8s.set_config(label_config), worker.set_config(label_config)) await kubernetes_cluster.wait_for_idle(status="active", timeout=10 * 60)