Adds ability to set extra args for kube services

charms/worker/charmcraft.yaml

@@ -69,6 +69,34 @@ config:
         
         Note: Due to NodeRestriction, workers are limited to how they can label themselves
         https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#noderestriction
+    kube-proxy-extra-args:
+      type: string
+      default: ""
+      description: |
+        Space separated list of flags and key=value pairs that will be passed as arguments to
+        kube-proxy.
+
+        Notes:
+          Options may only be set on charm deployment
+
+        For example a value like this:
+          runtime-config=batch/v2alpha1=true profiling=true
+        will result in kube-proxy being run with the following options:
+          --runtime-config=batch/v2alpha1=true --profiling=true
+    kubelet-extra-args:
+      type: string
+      default: ""
+      description: |
+        Space separated list of flags and key=value pairs that will be passed as arguments to
+        kubelet.
+
+        Notes:
+          Options may only be set on charm deployment
+
+        For example a value like this:
+          runtime-config=batch/v2alpha1=true profiling=true
+        will result in kubelet being run with the following options:
+          --runtime-config=batch/v2alpha1=true --profiling=true
 
 resources:
   snap-installation:

charms/worker/k8s/charmcraft.yaml

@@ -210,6 +210,77 @@ config:
       default: false
       description: |
         Enable/Disable the gateway feature on the cluster.
+    kube-apiserver-extra-args:
+      type: string
+      default: ""
+      description: |
+        Space separated list of flags and key=value pairs that will be passed as arguments to
+        kube-apiserver.
+
+        Notes:
+          Options may only be set on charm deployment
+
+        For example a value like this:
+          runtime-config=batch/v2alpha1=true profiling=true
+        will result in kube-apiserver being run with the following options:
+          --runtime-config=batch/v2alpha1=true --profiling=true
+    kube-controller-manager-extra-args:
+      type: string
+      default: ""
+      description: |
+        Space separated list of flags and key=value pairs that will be passed as arguments to
+        kube-controller-manager.
+
+        Notes:
+          Options may only be set on charm deployment
+          cluster-name: cannot be overridden
+
+        For example a value like this:
+          runtime-config=batch/v2alpha1=true profiling=true
+        will result in kube-controller-manager being run with the following options:
+          --runtime-config=batch/v2alpha1=true --profiling=true
+    kube-proxy-extra-args:
+      type: string
+      default: ""
+      description: |
+        Space separated list of flags and key=value pairs that will be passed as arguments to
+        kube-proxy.
+
+        Notes:
+          Options may only be set on charm deployment
+
+        For example a value like this:
+          runtime-config=batch/v2alpha1=true profiling=true
+        will result in kube-proxy being run with the following options:
+          --runtime-config=batch/v2alpha1=true --profiling=true
+    kube-scheduler-extra-args:
+      type: string
+      default: ""
+      description: |
+        Space separated list of flags and key=value pairs that will be passed as arguments to
+        kube-scheduler.
+
+        Notes:
+          Options may only be set on charm deployment
+
+        For example a value like this:
+          runtime-config=batch/v2alpha1=true profiling=true
+        will result in kube-scheduler being run with the following options:
+          --runtime-config=batch/v2alpha1=true --profiling=true
+    kubelet-extra-args:
+      type: string
+      default: ""
+      description: |
+        Space separated list of flags and key=value pairs that will be passed as arguments to
+        kubelet.
+
+        Notes:
+          Options may only be set on charm deployment
+
+        For example a value like this:
+          runtime-config=batch/v2alpha1=true profiling=true
+        will result in kubelet being run with the following options:
+          --runtime-config=batch/v2alpha1=true --profiling=true
     load-balancer-enabled:
       type: boolean
       default: false

charms/worker/k8s/lib/charms/k8s/v0/k8sd_api_manager.py

@@ -45,7 +45,7 @@ class utilises different connection factories (UnixSocketConnectionFactory
 
 # Increment this PATCH version before using `charmcraft publish-lib` or reset
 # to 0 if you are raising the major API version
-LIBPATCH = 3
+LIBPATCH = 4
 
 logger = logging.getLogger(__name__)
 
@@ -357,7 +357,11 @@ class BootstrapConfig(BaseModel):
         datastore_client_cert (str): The client certificate for accessing the datastore.
         datastore_client_key (str): The client key for accessing the datastore.
         extra_sans (List[str]): List of extra sans for the self-signed certificates
+        extra_node_kube_apiserver_args ([Dict[str,str]]): key-value service args
         extra_node_kube_controller_manager_args ([Dict[str,str]]): key-value service args
+        extra_node_kube_scheduler_args ([Dict[str,str]]): key-value service args
+        extra_node_kube_proxy_args ([Dict[str,str]]): key-value service args
+        extra_node_kubelet_args ([Dict[str,str]]): key-value service args
     """
 
     cluster_config: Optional[UserFacingClusterConfig] = Field(default=None, alias="cluster-config")
@@ -373,9 +377,21 @@ class BootstrapConfig(BaseModel):
     datastore_client_cert: Optional[str] = Field(default=None, alias="datastore-client-crt")
     datastore_client_key: Optional[str] = Field(default=None, alias="datastore-client-key")
     extra_sans: Optional[List[str]] = Field(default=None, alias="extra-sans")
+    extra_node_kube_apiserver_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kube-apiserver-args"
+    )
     extra_node_kube_controller_manager_args: Optional[Dict[str, str]] = Field(
         default=None, alias="extra-node-kube-controller-manager-args"
     )
+    extra_node_kube_scheduler_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kube-scheduler-args"
+    )
+    extra_node_kube_proxy_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kube-proxy-args"
+    )
+    extra_node_kubelet_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kubelet-args"
+    )
 
 
 class CreateClusterRequest(BaseModel):
@@ -410,10 +426,18 @@ class NodeJoinConfig(BaseModel, allow_population_by_field_name=True):
     Attributes:
         kubelet_crt (str): node's certificate
         kubelet_key (str): node's certificate key
+        extra_node_kube_proxy_args (Dict[str, str]): key-value service args
+        extra_node_kubelet_args (Dict[str, str]): key-value service args
     """
 
     kubelet_crt: Optional[str] = Field(default=None, alias="kubelet-crt")
     kubelet_key: Optional[str] = Field(default=None, alias="kubelet-key")
+    extra_node_kube_proxy_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kube-proxy-args"
+    )
+    extra_node_kubelet_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kubelet-args"
+    )
 
 
 class ControlPlaneNodeJoinConfig(NodeJoinConfig, allow_population_by_field_name=True):
@@ -425,6 +449,9 @@ class ControlPlaneNodeJoinConfig(NodeJoinConfig, allow_population_by_field_name=
         apiserver_client_key (str): apiserver certificate key
         front_proxy_client_crt (str): front-proxy certificate
         front_proxy_client_key (str): front-proxy certificate key
+        extra_node_kube_apiserver_args (Dict[str, str]): key-value service args
+        extra_node_kube_controller_manager_args (Dict[str, str]): key-value service args
+        extra_node_kube_scheduler_args (Dict[str, str]): key-value service args
     """
 
     extra_sans: Optional[List[str]] = Field(default=None, alias="extra-sans")
@@ -433,6 +460,15 @@ class ControlPlaneNodeJoinConfig(NodeJoinConfig, allow_population_by_field_name=
     apiserver_client_key: Optional[str] = Field(default=None, alias="apiserver-key")
     front_proxy_client_crt: Optional[str] = Field(default=None, alias="front-proxy-client-crt")
     front_proxy_client_key: Optional[str] = Field(default=None, alias="front-proxy-client-key")
+    extra_node_kube_apiserver_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kube-apiserver-args"
+    )
+    extra_node_kube_controller_manager_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kube-controller-manager-args"
+    )
+    extra_node_kube_scheduler_args: Optional[Dict[str, str]] = Field(
+        default=None, alias="extra-node-kube-scheduler-args"
+    )
 
 
 class JoinClusterRequest(BaseModel, allow_population_by_field_name=True):

charms/worker/k8s/src/charm.py

@@ -49,6 +49,7 @@
     LocalStorageConfig,
     MetricsServerConfig,
     NetworkConfig,
+    NodeJoinConfig,
     UnixSocketConnectionFactory,
     UpdateClusterConfigRequest,
     UserFacingClusterConfig,
@@ -109,6 +110,58 @@ def _cluster_departing_unit(event: ops.EventBase) -> Union[Literal[False], ops.U
     )
 
 
+class ArgsMapping:
+    """A class to map string key,val pairs to be used as cmd args.
+
+    Attributes:
+        base_args: the base args to be used
+        extra_args: additional args to be used
+    """
+
+    def __init__(self, *_, **kwargs: str):
+        """Initialise the ArgsMapping class.
+
+        Args:
+            kwargs: the base args to be used
+        """
+        self.base_args = {k.lstrip("-"): v for k, v in kwargs.items()}
+        self.extra_args: Dict[str, str] = {}
+
+    def dict(self) -> Dict[str, str]:
+        """Return an args based representative view.
+
+        Returns:
+            Dict[str, str]: the args as a dictionary
+        """
+        args = {**self.base_args, **self.extra_args}
+        return {f"--{k}": v for k, v in args.items() if v is not None}
+
+    def parse(self, as_str: str):
+        """Parse the given string into extra_args.
+
+        Args:
+            as_str (str): the extra args to be added
+        """
+        elements = as_str.split()
+        args = {}
+
+        for element in elements:
+            if "=" in element:
+                key, _, value = element.partition("=")
+                args[key] = value
+            else:
+                args[element] = "true"
+        self.update(**args)
+
+    def update(self, *_, **kwargs: str):
+        """Update the extra_args with the given kwargs.
+
+        Args:
+            kwargs: the extra args to be added
+        """
+        self.extra_args.update({k.lstrip("-"): v for k, v in kwargs.items()})
+
+
 class NodeRemovedError(Exception):
     """Raised to prevent reconciliation of dying node."""
 
@@ -314,6 +367,48 @@ def _check_k8sd_ready(self):
         status.add(ops.MaintenanceStatus("Ensuring snap readiness"))
         self.api_manager.check_k8sd_ready()
 
+    def _extra_arguments(
+        self, config: Union[BootstrapConfig, ControlPlaneNodeJoinConfig, NodeJoinConfig]
+    ):
+        """Set extra arguments for Kubernetes components based on the provided configuration.
+
+        Updates the following attributes of the `config` object:
+            - extra_node_kube_apiserver_args: arguments for kube-apiserver.
+            - extra_node_kube_controller_manager_args: arguments for kube-controller-manager.
+            - extra_node_kube_scheduler_args: arguments for kube-scheduler.
+            - extra_node_kube_proxy_args: arguments for kube-proxy.
+            - extra_node_kubelet_args: arguments for kubelet.
+
+        Args:
+            config (Union[BootstrapConfig, ControlPlaneNodeJoinConfig, NodeJoinConfig]):
+                The configuration object to be updated with extra arguments.
+        """
+        if isinstance(config, (BootstrapConfig, ControlPlaneNodeJoinConfig)):
+            kube_api_server_args = ArgsMapping()
+            kube_api_server_args.parse(str(self.config["kube-apiserver-extra-args"]))
+            config.extra_node_kube_apiserver_args = kube_api_server_args.dict()
+
+            kube_controller_manager_args = ArgsMapping()
+            kube_controller_manager_args.parse(
+                str(self.config["kube-controller-manager-extra-args"])
+            )
+            kube_controller_manager_args.update(
+                **{"cluster-name": self._generate_unique_cluster_name()}
+            )
+            config.extra_node_kube_controller_manager_args = kube_controller_manager_args.dict()
+
+            kube_scheduler_args = ArgsMapping()
+            kube_scheduler_args.parse(str(self.config["kube-scheduler-extra-args"]))
+            config.extra_node_kube_scheduler_args = kube_scheduler_args.dict()
+
+        kube_proxy_args = ArgsMapping()
+        kube_proxy_args.parse(str(self.config["kube-proxy-extra-args"]))
+        config.extra_node_kube_proxy_args = kube_proxy_args.dict()
+
+        kubelet_args = ArgsMapping()
+        kubelet_args.parse(str(self.config["kubelet-extra-args"]))
+        config.extra_node_kubelet_args = kubelet_args.dict()
+
     @on_error(
         ops.WaitingStatus("Waiting to bootstrap k8s snap"),
         ReconcilerError,
@@ -333,9 +428,7 @@ def _bootstrap_k8s_snap(self):
         bootstrap_config.pod_cidr = str(self.config["bootstrap-pod-cidr"])
         bootstrap_config.control_plane_taints = str(self.config["bootstrap-node-taints"]).split()
         bootstrap_config.extra_sans = [_get_public_address()]
-        bootstrap_config.extra_node_kube_controller_manager_args = {
-            "--cluster-name": self._generate_unique_cluster_name()
-        }
+        self._extra_arguments(bootstrap_config)
 
         status.add(ops.MaintenanceStatus("Bootstrapping Cluster"))
 
@@ -744,6 +837,10 @@ def _join_with_token(self, relation: ops.Relation, token: str):
         if self.is_control_plane:
             request.config = ControlPlaneNodeJoinConfig()
             request.config.extra_sans = [_get_public_address()]
+            self._extra_arguments(request.config)
+        else:
+            request.config = NodeJoinConfig()
+            self._extra_arguments(request.config)
 
         self.api_manager.join_cluster(request)
         log.info("Joined %s(%s)", self.unit, node_name)

tests/integration/data/test-bundle.yaml

@@ -14,10 +14,18 @@ applications:
     expose: true
     options:
       bootstrap-node-taints: "node-role.kubernetes.io/control-plane=:NoSchedule"
+      kube-apiserver-extra-args: "v=3"
+      kube-controller-manager-extra-args: "v=3"
+      kube-proxy-extra-args: "v=3"
+      kube-scheduler-extra-args: "v=3"
+      kubelet-extra-args: "v=3"
   k8s-worker:
     charm: k8s-worker
     channel: latest/edge
     num_units: 2
     constraints: cores=2 mem=8G root-disk=16G
+    options:
+      kube-proxy-extra-args: "v=3"
+      kubelet-extra-args: "v=3"
 relations:
   - [k8s, k8s-worker:cluster]

tests/integration/test_k8s.py

@@ -176,6 +176,24 @@ async def override_snap_on_k8s(kubernetes_cluster: model.Model, request):
         await kubernetes_cluster.wait_for_idle(status="active", timeout=1 * 60)
 
 
+async def test_verbose_config(kubernetes_cluster: model.Model):
+    """Test verbose config."""
+    k8s = kubernetes_cluster.applications["k8s"]
+    worker = kubernetes_cluster.applications["k8s-worker"]
+    all_units = k8s.units + worker.units
+
+    unit_events = await asyncio.gather(*(u.run("ps axf | grep kube") for u in all_units))
+    unit_runs = await asyncio.gather(*(u.wait() for u in unit_events))
+    for idx, unit_run in enumerate(unit_runs):
+        rc, stdout, stderr = (
+            unit_run.results["return-code"],
+            unit_run.results["stdout"],
+            unit_run.results["stderr"],
+        )
+        assert rc == 0, f"Failed to run 'ps axf' on {all_units[idx].name}: {stderr}"
+        assert all("--v=3" for line in stdout.splitlines() if " /snap/k8s" in line)
+
+
 @pytest.mark.abort_on_fail
 async def test_override_snap_resource(override_snap_on_k8s: application.Application):
     """Override snap resource."""