From 11a94be0ac31645f9983cae7e2eef7cc3a36199a Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Sat, 20 Apr 2024 07:48:54 -0500 Subject: [PATCH 1/4] Join nodes and Bootstrap nodes with extra-sans config (#73) * Join nodes and Bootstrap nodes with extra-sans config * Use new datastore status --- .../k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 177 ++++++++++++------ charms/worker/k8s/src/charm.py | 24 ++- .../k8s/tests/unit/test_k8sd_api_manager.py | 31 ++- pyproject.toml | 1 + test_requirements.txt | 1 + tests/integration/test_etcd.py | 2 +- 6 files changed, 172 insertions(+), 64 deletions(-) diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index 23dd2335..30adec05 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -31,8 +31,9 @@ class utilises different connection factories (UnixSocketConnectionFactory import socket from contextlib import contextmanager from http.client import HTTPConnection, HTTPException -from typing import Generator, List, Optional, Type, TypeVar +from typing import Any, Dict, Generator, List, Optional, Type, TypeVar +import yaml from pydantic import AnyHttpUrl, BaseModel, Field, SecretStr, validator # The unique Charmhub library identifier, never change it @@ -162,8 +163,8 @@ class ClusterMember(BaseModel): name: str address: str - cluster_role: str = Field(None, alias="cluster-role") - datastore_role: str = Field(None, alias="datastore-role") + cluster_role: Optional[str] = Field(None, alias="cluster-role") + datastore_role: Optional[str] = Field(None, alias="datastore-role") class DNSConfig(BaseModel): @@ -176,10 +177,10 @@ class DNSConfig(BaseModel): upstream_nameservers: List of upstream nameservers for DNS resolution. """ - enabled: bool = Field(None) - cluster_domain: str = Field(None, alias="cluster-domain") - service_ip: str = Field(None, alias="service-ip") - upstream_nameservers: List[str] = Field(None, alias="upstream-nameservers") + enabled: Optional[bool] = Field(None) + cluster_domain: Optional[str] = Field(None, alias="cluster-domain") + service_ip: Optional[str] = Field(None, alias="service-ip") + upstream_nameservers: Optional[List[str]] = Field(None, alias="upstream-nameservers") class IngressConfig(BaseModel): @@ -191,9 +192,9 @@ class IngressConfig(BaseModel): enable_proxy_protocol: Optional flag to enable or disable proxy protocol. """ - enabled: bool = Field(None) - default_tls_secret: str = Field(None, alias="default-tls-secret") - enable_proxy_protocol: bool = Field(None, alias="enable-proxy-protocol") + enabled: Optional[bool] = Field(None) + default_tls_secret: Optional[str] = Field(None, alias="default-tls-secret") + enable_proxy_protocol: Optional[bool] = Field(None, alias="enable-proxy-protocol") class LoadBalancerConfig(BaseModel): @@ -211,15 +212,15 @@ class LoadBalancerConfig(BaseModel): bgp_peer_port: The port for BGP peering. """ - enabled: bool = Field(None) - cidrs: List[str] = Field(None) - l2_enabled: bool = Field(None, alias="l2-enabled") - l2_interfaces: List[str] = Field(None, alias="l2-interfaces") - bgp_enabled: bool = Field(None, alias="bgp-enabled") - bgp_local_asn: int = Field(None, alias="bgp-local-asn") - bgp_peer_address: str = Field(None, alias="bgp-peer-address") - bgp_peer_asn: int = Field(None, alias="bgp-peer-asn") - bgp_peer_port: int = Field(None, alias="bgp-peer-port") + enabled: Optional[bool] = Field(None) + cidrs: Optional[List[str]] = Field(None) + l2_enabled: Optional[bool] = Field(None, alias="l2-enabled") + l2_interfaces: Optional[List[str]] = Field(None, alias="l2-interfaces") + bgp_enabled: Optional[bool] = Field(None, alias="bgp-enabled") + bgp_local_asn: Optional[int] = Field(None, alias="bgp-local-asn") + bgp_peer_address: Optional[str] = Field(None, alias="bgp-peer-address") + bgp_peer_asn: Optional[int] = Field(None, alias="bgp-peer-asn") + bgp_peer_port: Optional[int] = Field(None, alias="bgp-peer-port") class LocalStorageConfig(BaseModel): @@ -232,10 +233,10 @@ class LocalStorageConfig(BaseModel): set_default: Optional flag to set this as the default storage option. """ - enabled: bool = Field(None) - local_path: str = Field(None, alias="local-path") - reclaim_policy: str = Field(None, alias="reclaim-policy") - set_default: bool = Field(None, alias="set-default") + enabled: Optional[bool] = Field(None) + local_path: Optional[str] = Field(None, alias="local-path") + reclaim_policy: Optional[str] = Field(None, alias="reclaim-policy") + set_default: Optional[bool] = Field(None, alias="set-default") class NetworkConfig(BaseModel): @@ -245,7 +246,7 @@ class NetworkConfig(BaseModel): enabled: Optional flag which represents the status of Network. """ - enabled: bool = Field(None) + enabled: Optional[bool] = Field(None) class GatewayConfig(BaseModel): @@ -255,7 +256,7 @@ class GatewayConfig(BaseModel): enabled: Optional flag which represents the status of Gateway. """ - enabled: bool = Field(None) + enabled: Optional[bool] = Field(None) class MetricsServerConfig(BaseModel): @@ -265,7 +266,7 @@ class MetricsServerConfig(BaseModel): enabled: Optional flag which represents the status of MetricsServer. """ - enabled: bool = Field(None) + enabled: Optional[bool] = Field(None) class UserFacingClusterConfig(BaseModel): @@ -282,14 +283,14 @@ class UserFacingClusterConfig(BaseModel): cloud_provider: The cloud provider for the cluster. """ - network: NetworkConfig = Field(None) - dns: DNSConfig = Field(None) - ingress: IngressConfig = Field(None) - load_balancer: LoadBalancerConfig = Field(None, alias="load-balancer") - local_storage: LocalStorageConfig = Field(None, alias="local-storage") - gateway: GatewayConfig = Field(None) - metrics_server: MetricsServerConfig = Field(None, alias="metrics-server") - cloud_provider: str = Field(None, alias="cloud-provider") + network: Optional[NetworkConfig] = Field(None) + dns: Optional[DNSConfig] = Field(None) + ingress: Optional[IngressConfig] = Field(None) + load_balancer: Optional[LoadBalancerConfig] = Field(None, alias="load-balancer") + local_storage: Optional[LocalStorageConfig] = Field(None, alias="local-storage") + gateway: Optional[GatewayConfig] = Field(None) + metrics_server: Optional[MetricsServerConfig] = Field(None, alias="metrics-server") + cloud_provider: Optional[str] = Field(None, alias="cloud-provider") class BootstrapConfig(BaseModel): @@ -307,19 +308,21 @@ class BootstrapConfig(BaseModel): datastore_ca_cert (str): The CA certificate for the datastore. datastore_client_cert (str): The client certificate for accessing the datastore. datastore_client_key (str): The client key for accessing the datastore. + extra_sans (List[str]): List of extra sans for the self-signed certificates """ - cluster_config: UserFacingClusterConfig = Field(None, alias="cluster-config") - pod_cidr: str = Field(None, alias="pod-cidr") - service_cidr: str = Field(None, alias="service-cidr") - disable_rbac: bool = Field(None, alias="disable-rbac") - secure_port: int = Field(None, alias="secure-port") - k8s_dqlite_port: int = Field(None, alias="k8s-dqlite-port") - datastore_type: str = Field(None, alias="datastore-type") - datastore_servers: List[AnyHttpUrl] = Field(None, alias="datastore-servers") - datastore_ca_cert: str = Field(None, alias="datastore-ca-crt") - datastore_client_cert: str = Field(None, alias="datastore-client-crt") - datastore_client_key: str = Field(None, alias="datastore-client-key") + cluster_config: Optional[UserFacingClusterConfig] = Field(None, alias="cluster-config") + pod_cidr: Optional[str] = Field(None, alias="pod-cidr") + service_cidr: Optional[str] = Field(None, alias="service-cidr") + disable_rbac: Optional[bool] = Field(None, alias="disable-rbac") + secure_port: Optional[int] = Field(None, alias="secure-port") + k8s_dqlite_port: Optional[int] = Field(None, alias="k8s-dqlite-port") + datastore_type: Optional[str] = Field(None, alias="datastore-type") + datastore_servers: Optional[List[AnyHttpUrl]] = Field(None, alias="datastore-servers") + datastore_ca_cert: Optional[str] = Field(None, alias="datastore-ca-crt") + datastore_client_cert: Optional[str] = Field(None, alias="datastore-client-crt") + datastore_client_key: Optional[str] = Field(None, alias="datastore-client-key") + extra_sans: Optional[List[str]] = Field(None, alias="extra-sans") class CreateClusterRequest(BaseModel): @@ -346,6 +349,68 @@ class UpdateClusterConfigRequest(BaseModel): config: UserFacingClusterConfig +class NodeJoinConfig(BaseModel, allow_population_by_field_name=True): + """Request model for the config on a node joining the cluster. + + Attributes: + kubelet_crt (str): node's certificate + kubelet_key (str): node's certificate key + """ + + kubelet_crt: Optional[str] = Field(None, alias="kubelet-crt") + kubelet_key: Optional[str] = Field(None, alias="kubelet-key") + + +class ControlPlaneNodeJoinConfig(NodeJoinConfig, allow_population_by_field_name=True): + """Request model for the config on a control-plane node joining the cluster. + + Attributes: + extra_sans (List[str]): List of extra sans for the self-signed certificates + apiserver_crt (str): apiserver certificate + apiserver_client_key (str): apiserver certificate key + front_proxy_client_crt (str): front-proxy certificate + front_proxy_client_key (str): front-proxy certificate key + """ + + extra_sans: Optional[List[str]] = Field(None, alias="extra-sans") + + apiserver_crt: Optional[str] = Field(None, alias="apiserver-crt") + apiserver_client_key: Optional[str] = Field(None, alias="apiserver-key") + front_proxy_client_crt: Optional[str] = Field(None, alias="front-proxy-client-crt") + front_proxy_client_key: Optional[str] = Field(None, alias="front-proxy-client-key") + + +class JoinClusterRequest(BaseModel, allow_population_by_field_name=True): + """Request model for a node joining the cluster. + + Attributes: + name (str): node's certificate + address (str): node's certificate key + token (str): token + config (NodeJoinConfig): Node Config + """ + + name: str + address: str + token: SecretStr + config: Optional[NodeJoinConfig] = Field(None) + + def dict(self, **kwds) -> Dict[Any, Any]: + """Render object into a dict. + + Arguments: + kwds: keyword arguments + + Returns: + dict mapping of the object + """ + rendered = super().dict(**kwds) + rendered["token"] = self.token.get_secret_value() + if self.config: + rendered["config"] = yaml.safe_dump(self.config.dict(**kwds)) + return rendered + + class DatastoreStatus(BaseModel): """information regarding the active datastore. @@ -354,8 +419,8 @@ class DatastoreStatus(BaseModel): external_url: (str): list of external_urls """ - datastore_type: str = Field(None, alias="type") - external_url: str = Field(None, alias="external-url") + datastore_type: Optional[str] = Field(None, alias="type") + external_url: Optional[str] = Field(None, alias="external-url") class ClusterStatus(BaseModel): @@ -369,9 +434,9 @@ class ClusterStatus(BaseModel): """ ready: bool = Field(False) - members: List[ClusterMember] = Field(None) - config: UserFacingClusterConfig = Field(None) - datastore: DatastoreStatus = Field(None) + members: Optional[List[ClusterMember]] = Field(None) + config: Optional[UserFacingClusterConfig] = Field(None) + datastore: Optional[DatastoreStatus] = Field(None) class ClusterMetadata(BaseModel): @@ -392,7 +457,7 @@ class GetClusterStatusResponse(BaseRequestModel): Can be None if the status is not available. """ - metadata: ClusterMetadata = Field(None) + metadata: Optional[ClusterMetadata] = Field(None) class KubeConfigMetadata(BaseModel): @@ -597,17 +662,15 @@ def create_join_token(self, name: str, worker: bool = False) -> SecretStr: join_response = self._send_request(endpoint, "POST", CreateJoinTokenResponse, body) return join_response.metadata.token - def join_cluster(self, name: str, address: str, token: str): + def join_cluster(self, config: JoinClusterRequest): """Join a node to the k8s cluster. Args: - name (str): Name of the node. - address (str): address to which k8sd should be bound - token (str): The join token for this node. + config: JoinClusterRequest: config to join the cluster """ endpoint = "/1.0/k8sd/cluster/join" - body = {"name": name, "address": address, "token": token} - self._send_request(endpoint, "POST", EmptyResponse, body) + request = config.dict(exclude_none=True, by_alias=True) + self._send_request(endpoint, "POST", EmptyResponse, request) def remove_node(self, name: str, force: bool = True): """Remove a node from the cluster. diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index c346cc04..a0fda2b2 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -34,9 +34,11 @@ from charms.grafana_agent.v0.cos_agent import COSAgentProvider from charms.k8s.v0.k8sd_api_manager import ( BootstrapConfig, + ControlPlaneNodeJoinConfig, CreateClusterRequest, DNSConfig, InvalidResponseError, + JoinClusterRequest, K8sdAPIManager, K8sdConnectionError, NetworkConfig, @@ -64,6 +66,16 @@ SUPPORTED_DATASTORES = ["dqlite", "etcd"] +def _get_public_address() -> str: + """Get public address from juju. + + Returns: + (str) public ip address of the unit + """ + cmd = ["unit-get", "public-address"] + return subprocess.check_output(cmd).decode("UTF-8").strip() + + class K8sCharm(ops.CharmBase): """A charm for managing a K8s cluster via the k8s snap. @@ -221,6 +233,7 @@ def _bootstrap_k8s_snap(self): bootstrap_config = BootstrapConfig() self._configure_datastore(bootstrap_config) + bootstrap_config.extra_sans = [_get_public_address()] status.add(ops.MaintenanceStatus("Bootstrapping Cluster")) @@ -446,7 +459,12 @@ def _join_cluster(self): node_name = self.get_node_name() cluster_addr = f"{address}:{K8SD_PORT}" log.info("Joining %s to %s...", node_name, cluster_addr) - self.api_manager.join_cluster(node_name, cluster_addr, token) + request = JoinClusterRequest(name=node_name, address=cluster_addr, token=token) + if self.is_control_plane: + request.config = ControlPlaneNodeJoinConfig() + request.config.extra_sans = [_get_public_address()] + + self.api_manager.join_cluster(request) log.info("Success") def _reconcile(self, event): @@ -582,9 +600,7 @@ def _get_external_kubeconfig(self, event: ops.ActionEvent): server = event.params.get("server") if not server: log.info("No server requested, use public-address") - cmd = ["unit-get", "public-address"] - addr = subprocess.check_output(cmd).decode("UTF-8").strip() - server = f"{addr}:6443" + server = f"{_get_public_address()}:6443" log.info("Requesting kubeconfig for server=%s", server) resp = self.api_manager.get_kubeconfig(server) event.set_results({"kubeconfig": resp}) diff --git a/charms/worker/k8s/tests/unit/test_k8sd_api_manager.py b/charms/worker/k8s/tests/unit/test_k8sd_api_manager.py index 875a5e9d..3f785de8 100644 --- a/charms/worker/k8s/tests/unit/test_k8sd_api_manager.py +++ b/charms/worker/k8s/tests/unit/test_k8sd_api_manager.py @@ -13,11 +13,13 @@ AuthTokenResponse, BaseRequestModel, BootstrapConfig, + ControlPlaneNodeJoinConfig, CreateClusterRequest, CreateJoinTokenResponse, DNSConfig, EmptyResponse, InvalidResponseError, + JoinClusterRequest, K8sdAPIManager, K8sdConnectionError, NetworkConfig, @@ -233,11 +235,36 @@ def test_create_join_token_worker(self, mock_send_request): ) @patch("lib.charms.k8s.v0.k8sd_api_manager.K8sdAPIManager._send_request") - def test_join_cluster(self, mock_send_request): + def test_join_cluster_control_plane(self, mock_send_request): """Test successfully joining a cluster.""" mock_send_request.return_value = EmptyResponse(status_code=200, type="test", error_code=0) - self.api_manager.join_cluster("test-node", "127.0.0.1:6400", "test-token") + request = JoinClusterRequest( + name="test-node", address="127.0.0.1:6400", token="test-token" + ) + request.config = ControlPlaneNodeJoinConfig(extra_sans=["127.0.0.1"]) + self.api_manager.join_cluster(request) + mock_send_request.assert_called_once_with( + "/1.0/k8sd/cluster/join", + "POST", + EmptyResponse, + { + "name": "test-node", + "address": "127.0.0.1:6400", + "token": "test-token", + "config": "extra-sans:\n- 127.0.0.1\n", + }, + ) + + @patch("lib.charms.k8s.v0.k8sd_api_manager.K8sdAPIManager._send_request") + def test_join_cluster_worker(self, mock_send_request): + """Test successfully joining a cluster.""" + mock_send_request.return_value = EmptyResponse(status_code=200, type="test", error_code=0) + + request = JoinClusterRequest( + name="test-node", address="127.0.0.1:6400", token="test-token" + ) + self.api_manager.join_cluster(request) mock_send_request.assert_called_once_with( "/1.0/k8sd/cluster/join", "POST", diff --git a/pyproject.toml b/pyproject.toml index 890e99bd..530a4a95 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ profile = "black" ignore_missing_imports = true explicit_package_bases = true namespace_packages = true +plugins = "pydantic.mypy" [tool.pylint] # Ignore too-few-public-methods due to pydantic models diff --git a/test_requirements.txt b/test_requirements.txt index 926f7e74..4906b370 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,5 @@ juju +pydantic<2 pylxd pytest pytest-asyncio diff --git a/tests/integration/test_etcd.py b/tests/integration/test_etcd.py index f2936d7d..db015f96 100644 --- a/tests/integration/test_etcd.py +++ b/tests/integration/test_etcd.py @@ -40,4 +40,4 @@ async def test_etcd_datastore(kubernetes_cluster: model.Model): status = json.loads(result.results["stdout"]) assert status["ready"], "Cluster isn't ready" assert status["datastore"]["type"] == "external", "Not bootstrapped against etcd" - assert status["datastore"]["external-url"] == f"https://{etcd.public_address}:{etcd_port}" + assert status["datastore"]["servers"] == [f"https://{etcd.public_address}:{etcd_port}"] From efff0cb167b17c13fc213f014bc7c2104d575436 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 20 Apr 2024 09:01:41 -0500 Subject: [PATCH 2/4] chore: update charm libraries (#67) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Adam Dyess --- .../worker/k8s/lib/charms/operator_libs_linux/v2/snap.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/charms/worker/k8s/lib/charms/operator_libs_linux/v2/snap.py b/charms/worker/k8s/lib/charms/operator_libs_linux/v2/snap.py index 871ff5de..ef426775 100644 --- a/charms/worker/k8s/lib/charms/operator_libs_linux/v2/snap.py +++ b/charms/worker/k8s/lib/charms/operator_libs_linux/v2/snap.py @@ -83,7 +83,7 @@ # Increment this PATCH version before using `charmcraft publish-lib` or reset # to 0 if you are raising the major API version -LIBPATCH = 4 +LIBPATCH = 5 # Regex to locate 7-bit C1 ANSI sequences @@ -580,10 +580,17 @@ def ensure( # We are installing or refreshing a snap. if self._state not in (SnapState.Present, SnapState.Latest): # The snap is not installed, so we install it. + logger.info( + "Installing snap %s, revision %s, tracking %s", self._name, revision, channel + ) self._install(channel, cohort, revision) else: # The snap is installed, but we are changing it (e.g., switching channels). + logger.info( + "Refreshing snap %s, revision %s, tracking %s", self._name, revision, channel + ) self._refresh(channel=channel, cohort=cohort, revision=revision, devmode=devmode) + logger.info("The snap installation completed successfully") self._update_snap_apps() self._state = state From c59f2913d2d052e3044bd93dbbeab78621a0714c Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Sat, 20 Apr 2024 12:37:42 -0500 Subject: [PATCH 3/4] Improve the juju unit status (#72) * Improve the juju unit status * Apply suggestions from code review Co-authored-by: Mateo Florido <32885896+mateoflorido@users.noreply.github.com> * update test due to k8s status change --------- Co-authored-by: Mateo Florido <32885896+mateoflorido@users.noreply.github.com> --- charms/worker/k8s/src/charm.py | 39 +++++++++++------------ charms/worker/k8s/tests/unit/test_base.py | 5 ++- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index a0fda2b2..d06d0fca 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -140,7 +140,7 @@ def _apply_cos_requirements(self): return log.info("Apply COS Integrations") - status.add(ops.MaintenanceStatus("Configuring COS Integration")) + status.add(ops.MaintenanceStatus("Ensuring COS Integration")) subprocess.check_call(shlex.split("k8s kubectl apply -f templates/cos_roles.yaml")) subprocess.check_call(shlex.split("k8s kubectl apply -f templates/ksm.yaml")) @@ -196,18 +196,18 @@ def get_cloud_name(self) -> str: @on_error(ops.BlockedStatus("Failed to install k8s snap."), SnapError) def _install_k8s_snap(self): """Install the k8s snap package.""" - status.add(ops.MaintenanceStatus("Installing k8s snap")) + status.add(ops.MaintenanceStatus("Ensuring snap installation")) log.info("Ensuring k8s snap version") snap_ensure("k8s", SnapState.Latest.value, self.config["channel"]) - @on_error(WaitingStatus("Failed to apply snap requirements"), subprocess.CalledProcessError) + @on_error(WaitingStatus("Waiting to apply snap requirements"), subprocess.CalledProcessError) def _apply_snap_requirements(self): """Apply necessary snap requirements for the k8s snap. This method executes necessary scripts to ensure that the snap meets the network and interface requirements. """ - status.add(ops.MaintenanceStatus("Applying K8s requirements")) + status.add(ops.MaintenanceStatus("Ensuring snap requirements")) log.info("Applying K8s requirements") init_sh = "/snap/k8s/current/k8s/hack/init.sh" subprocess.check_call(shlex.split(init_sh)) @@ -216,7 +216,7 @@ def _apply_snap_requirements(self): def _check_k8sd_ready(self): """Check if k8sd is ready to accept requests.""" log.info("Check if k8ds is ready") - status.add(ops.MaintenanceStatus("Check k8sd ready")) + status.add(ops.MaintenanceStatus("Ensuring snap readiness")) self.api_manager.check_k8sd_ready() @on_error( @@ -247,18 +247,13 @@ def _bootstrap_k8s_snap(self): # TODO: Make port (and address) configurable. self.api_manager.bootstrap_k8s_snap(payload) - @status.on_error( - ops.WaitingStatus("Configuring COS Integration"), - subprocess.CalledProcessError, - AssertionError, - ) def _configure_cos_integration(self): """Retrieve the join token from secret databag and join the cluster.""" if not self.model.get_relation("cos-agent"): return - status.add(ops.MaintenanceStatus("Configuring COS integration")) - log.info("Configuring COS integration") + status.add(ops.MaintenanceStatus("Updating COS integrations")) + log.info("Updating COS integration") if relation := self.model.get_relation("cos-tokens"): self.collector.request(relation) @@ -359,14 +354,14 @@ def _create_cos_tokens(self): self.distributor.allocate_tokens(relation=rel, token_strategy=TokenStrategy.COS) @on_error( - WaitingStatus("Waiting for enable functionalities"), + WaitingStatus("Waiting to enable features"), InvalidResponseError, K8sdConnectionError, ) def _enable_functionalities(self): """Enable necessary components for the Kubernetes cluster.""" - status.add(ops.MaintenanceStatus("Enabling Functionalities")) - log.info("Enabling Functionalities") + status.add(ops.MaintenanceStatus("Updating K8s features")) + log.info("Enabling K8s features") dns_config = DNSConfig(enabled=True) network_config = NetworkConfig(enabled=True) user_cluster_config = UserFacingClusterConfig(dns=dns_config, network=network_config) @@ -513,10 +508,12 @@ def _update_status(self): status.add(ops.WaitingStatus("Preparing to leave cluster")) return if self.is_worker: - relation = self.model.get_relation("cluster") - assert relation, "Missing cluster relation with k8s" # nosec + if not self.model.get_relation("cluster"): + status.add(ops.BlockedStatus("Missing cluster integration")) + assert False, "Missing cluster integration" # nosec else: assert self.api_manager.is_cluster_ready(), "control-plane not yet ready" # nosec + if version := self._get_snap_version(): self.unit.set_workload_version(version) @@ -544,7 +541,7 @@ def _last_gasp(self, event): if not isinstance(event, ops.StopEvent): return busy_wait = 30 - status.add(ops.MaintenanceStatus("Awaiting cluster removal")) + status.add(ops.MaintenanceStatus("Ensuring cluster removal")) while busy_wait and self.api_manager.is_cluster_bootstrapped(): log.info("Waiting for this unit to uncluster") sleep(1) @@ -553,7 +550,7 @@ def _last_gasp(self, event): @status.on_error(ops.BlockedStatus("Cannot apply node-labels"), LabelMaker.NodeLabelError) def _apply_node_labels(self): """Apply labels to the node.""" - status.add(ops.MaintenanceStatus("Apply Node Labels")) + status.add(ops.MaintenanceStatus("Ensuring Kubernetes Node Labels")) node = self.get_node_name() if self.labeler.active_labels() is not None: self.labeler.apply_node_labels() @@ -569,7 +566,7 @@ def _on_update_status(self, _event: ops.UpdateStatusEvent): with status.context(self.unit): self._update_status() except status.ReconcilerError: - log.exception("Can't to update_status") + log.exception("Can't update_status") @property def _internal_kubeconfig(self) -> Path: @@ -579,7 +576,7 @@ def _internal_kubeconfig(self) -> Path: @on_error(ops.WaitingStatus("")) def _copy_internal_kubeconfig(self): """Write internal kubeconfig to /root/.kube/config.""" - status.add(ops.MaintenanceStatus("Generating KubeConfig")) + status.add(ops.MaintenanceStatus("Regenerating KubeConfig")) KUBECONFIG.parent.mkdir(parents=True, exist_ok=True) KUBECONFIG.write_bytes(self._internal_kubeconfig.read_bytes()) diff --git a/charms/worker/k8s/tests/unit/test_base.py b/charms/worker/k8s/tests/unit/test_base.py index 1317aa09..5dcef0f4 100644 --- a/charms/worker/k8s/tests/unit/test_base.py +++ b/charms/worker/k8s/tests/unit/test_base.py @@ -86,7 +86,10 @@ def test_update_status(harness): """ harness.charm.reconciler.stored.reconciled = True # Pretended to be reconciled harness.charm.on.update_status.emit() - assert harness.model.unit.status == ops.WaitingStatus("Cluster not yet ready") + if harness.charm.is_worker: + assert harness.model.unit.status == ops.BlockedStatus("Missing cluster integration") + else: + assert harness.model.unit.status == ops.WaitingStatus("Cluster not yet ready") def test_set_leader(harness): From 873c2637a2d0d43e416661c5a69918a14969e134 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Sat, 20 Apr 2024 13:10:38 -0500 Subject: [PATCH 4/4] Bootstrap with bootstrapConfig -- including registerWithTaints and service-cidr (#43) --- charms/worker/k8s/charmcraft.yaml | 22 ++++++++++++++++++- .../k8s/lib/charms/k8s/v0/k8sd_api_manager.py | 2 ++ charms/worker/k8s/src/charm.py | 2 ++ tests/integration/data/test-bundle.yaml | 2 +- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/charms/worker/k8s/charmcraft.yaml b/charms/worker/k8s/charmcraft.yaml index f83d801e..c2658e2e 100644 --- a/charms/worker/k8s/charmcraft.yaml +++ b/charms/worker/k8s/charmcraft.yaml @@ -67,7 +67,27 @@ config: type: string description: | Labels can be used to organize and to select subsets of nodes in the - cluster. Declare node labels in key=value format, separated by spaces. + cluster. Declare node labels in key=value format, separated by spaces. + register-with-taints: + type: string + default: "" + description: | + Space-separated list of taints to apply to this node at registration time. + + This config is only used at deploy time when Kubelet first registers the + node with Kubernetes. To change node taints after deploy time, use kubectl + instead. + + For more information, see the upstream Kubernetes documentation about + taints: + https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ + service-cidr: + type: string + default: 10.152.183.0/24 + description: | + CIDR to use for Kubernetes services. After deployment it is + only possible to increase the size of the IP range. It is not possible to + change or shrink the address range after deployment. actions: get-kubeconfig: diff --git a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py index 30adec05..27ef7cfa 100644 --- a/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py +++ b/charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py @@ -298,6 +298,7 @@ class BootstrapConfig(BaseModel): Attributes: cluster_config (UserFacingClusterConfig): The cluster configuration settings. + control_plane_taints (List[str]): Register with the following control-plane taints pod_cidr (str): The IP address range for the cluster's pods. service_cidr (str): The IP address range for the cluster services. disable_rbac (bool): Flag to disable role-based access control @@ -312,6 +313,7 @@ class BootstrapConfig(BaseModel): """ cluster_config: Optional[UserFacingClusterConfig] = Field(None, alias="cluster-config") + control_plane_taints: Optional[List[str]] = Field(None, alias="control-plane-taints") pod_cidr: Optional[str] = Field(None, alias="pod-cidr") service_cidr: Optional[str] = Field(None, alias="service-cidr") disable_rbac: Optional[bool] = Field(None, alias="disable-rbac") diff --git a/charms/worker/k8s/src/charm.py b/charms/worker/k8s/src/charm.py index d06d0fca..46f4466f 100755 --- a/charms/worker/k8s/src/charm.py +++ b/charms/worker/k8s/src/charm.py @@ -233,6 +233,8 @@ def _bootstrap_k8s_snap(self): bootstrap_config = BootstrapConfig() self._configure_datastore(bootstrap_config) + bootstrap_config.service_cidr = self.config["service-cidr"] + bootstrap_config.control_plane_taints = self.config["register-with-taints"].split() bootstrap_config.extra_sans = [_get_public_address()] status.add(ops.MaintenanceStatus("Bootstrapping Cluster")) diff --git a/tests/integration/data/test-bundle.yaml b/tests/integration/data/test-bundle.yaml index ce060fce..dcc5710f 100644 --- a/tests/integration/data/test-bundle.yaml +++ b/tests/integration/data/test-bundle.yaml @@ -4,7 +4,7 @@ name: integration-test description: |- Used to deploy or refresh within an integration test model -series: jammy +series: focal applications: k8s: charm: k8s