Merge pull request #689 from Netcracker/CPDEV_105612_k8s_1.31_adoption

[CPDEV_105612] k8s_1.31_adoption

documentation/Maintenance.md

@@ -752,11 +752,11 @@ The procedure recovers thirdparties based on the `cluster.yaml`. If rpm thirdpar
 restore_plan:
   thirdparties:
     /usr/bin/kubeadm:
-      source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubeadm
+      source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubeadm
     /usr/bin/kubelet:
-      source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubelet
+      source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubelet
     /usr/bin/kubectl:
-      source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubectl
+      source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubectl
     /usr/bin/calicoctl:
       source: https://github.com/projectcalico/calicoctl/releases/download/v3.14.1/calicoctl-linux-amd64
 ```

examples/procedure.yaml/full-restore.yaml

@@ -12,11 +12,11 @@ restore_plan:
       peer_cacert: /etc/kubernetes/pki/etcd/ca.crt
   thirdparties:
     /usr/bin/kubeadm:
-      source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubeadm
+      source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubeadm
     /usr/bin/kubelet:
-      source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubelet
+      source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubelet
     /usr/bin/kubectl:
-      source: https://storage.googleapis.com/kubernetes-release/release/v1.18.8/bin/linux/amd64/kubectl
+      source: https://dl.k8s.io/v1.18.8/bin/linux/amd64/kubectl
     /opt/cni/cni-plugins-linux.tgz:
       source: https://github.com/containernetworking/plugins/releases/download/v0.8.6/cni-plugins-linux-amd64-v0.8.6.tgz
     /usr/bin/calicoctl:

kubemarine/core/resources.py

@@ -462,6 +462,7 @@ def enrichment_functions(self) -> List[c.EnrichmentFunction]:
             # * kubemarine.sysctl.enrich_inventory
             kubemarine.kubernetes.enrich_inventory,
             kubemarine.admission.enrich_inventory,
+            kubemarine.kubernetes.enrich_control_plane_kubelet_local_mode,
             # Depends on kubemarine.core.defaults.apply_defaults
             kubemarine.kubernetes_accounts.enrich_inventory,
             # Depends on kubemarine.kubernetes.enrich_inventory

kubemarine/kubernetes/__init__.py

@@ -188,6 +188,18 @@ def enrich_inventory(cluster: KubernetesCluster) -> None:
     enrich_kube_proxy(cluster)
 
 
+@enrichment(EnrichmentStage.FULL)
+def enrich_control_plane_kubelet_local_mode(cluster: KubernetesCluster) -> None:
+    inventory = cluster.inventory
+
+    kubeadm = inventory["services"]["kubeadm"]
+    if components.control_plane_kubelet_local_mode(cluster):
+        feature_gates = kubeadm.get("featureGates", {})
+        if 'ControlPlaneKubeletLocalMode' not in feature_gates:
+            feature_gates['ControlPlaneKubeletLocalMode'] = True
+        kubeadm["featureGates"] = feature_gates
+
+
 def enrich_kube_proxy(cluster: KubernetesCluster) -> None:
     inventory = cluster.inventory
 

kubemarine/kubernetes/components.py

@@ -208,6 +208,10 @@ def kube_proxy_overwrites_higher_system_values(cluster: KubernetesCluster) -> bo
     return kubernetes_minor_release_at_least(cluster.inventory, "v1.29")
 
 
+def control_plane_kubelet_local_mode(cluster: KubernetesCluster) -> bool:
+    return kubernetes_minor_release_at_least(cluster.inventory, "v1.31")
+
+
 def kubernetes_minor_release_at_least(inventory: dict, minor_version: str) -> bool:
     kubernetes_version = inventory["services"]["kubeadm"]["kubernetesVersion"]
     return utils.version_key(kubernetes_version)[0:2] >= utils.minor_version_key(minor_version)

kubemarine/procedures/check_paas.py

@@ -1511,7 +1511,11 @@ def kubernetes_admission_status(cluster: KubernetesCluster) -> None:
 
         kubeadm_config = components.KubeadmConfig(cluster)
         cluster_config = kubeadm_config.load('kubeadm-config', first_control_plane)
+        # Check if extraArgs is a list of dictionaries or a dictionary
         apiserver_actual_args = cluster_config["apiServer"]["extraArgs"]
+        if isinstance(apiserver_actual_args, list):
+            # Convert list of dictionaries to a single dictionary
+            apiserver_actual_args = {arg['name']: arg['value'] for arg in apiserver_actual_args}
 
         actual_state = "disabled"
         if "admission-control-config-file" in apiserver_actual_args and (

kubemarine/procedures/upgrade.py

@@ -93,6 +93,20 @@ def edit_kube_proxy_conntrack_min(kube_proxy_cm: dict) -> dict:
         preconfigure_components.append('kube-proxy')
         preconfigure_functions['kube-proxy'] = edit_kube_proxy_conntrack_min
 
+    if kubernetes.components.control_plane_kubelet_local_mode(cluster):
+
+        # featureGates field is changed for kubernetes >= 1.31
+        # See kubernetes.enrich_control_plane_kubelet_local_mode() for details
+        def apply_kubelet_local_mode(cluster_config: dict) -> dict:
+            feature_gates = cluster.inventory["services"]["kubeadm"].get("featureGates")
+            if feature_gates is not None:
+                cluster_config["featureGates"] = feature_gates
+
+            return cluster_config
+
+        preconfigure_components.append('kube-apiserver')
+        preconfigure_functions['kubeadm-config'] = apply_kubelet_local_mode
+
     if preconfigure_components:
         upgrade_group.call(kubernetes.components.reconfigure_components,
                            components=preconfigure_components, edit_functions=preconfigure_functions)

kubemarine/resources/configurations/compatibility/internal/kubernetes_images.yaml

@@ -41,6 +41,8 @@ kube-apiserver:
     version: v1.30.1
   v1.30.3:
     version: v1.30.3
+  v1.31.1:
+    version: v1.31.1
 kube-controller-manager:
   v1.26.3:
     version: v1.26.3
@@ -82,6 +84,8 @@ kube-controller-manager:
     version: v1.30.1
   v1.30.3:
     version: v1.30.3
+  v1.31.1:
+    version: v1.31.1
 kube-scheduler:
   v1.26.3:
     version: v1.26.3
@@ -123,6 +127,8 @@ kube-scheduler:
     version: v1.30.1
   v1.30.3:
     version: v1.30.3
+  v1.31.1:
+    version: v1.31.1
 kube-proxy:
   v1.26.3:
     version: v1.26.3
@@ -164,6 +170,8 @@ kube-proxy:
     version: v1.30.1
   v1.30.3:
     version: v1.30.3
+  v1.31.1:
+    version: v1.31.1
 pause:
   v1.26.3:
     version: '3.9'
@@ -205,6 +213,8 @@ pause:
     version: '3.9'
   v1.30.3:
     version: '3.9'
+  v1.31.1:
+    version: '3.10'
 etcd:
   v1.26.3:
     version: 3.5.6-0
@@ -246,6 +256,8 @@ etcd:
     version: 3.5.12-0
   v1.30.3:
     version: 3.5.12-0
+  v1.31.1:
+    version: 3.5.15-0
 coredns/coredns:
   v1.26.3:
     version: v1.9.3
@@ -287,3 +299,5 @@ coredns/coredns:
     version: v1.11.1
   v1.30.3:
     version: v1.11.1
+  v1.31.1:
+    version: v1.11.3

kubemarine/resources/configurations/compatibility/internal/packages.yaml

@@ -45,6 +45,8 @@ containerd:
     version_debian: 1.6.*
   v1.30.3:
     version_debian: 1.6.*
+  v1.31.1:
+    version_debian: 1.6.*
 containerdio:
   v1.26.3:
     version_rhel: 1.6*
@@ -126,6 +128,10 @@ containerdio:
     version_rhel: 1.6*
     version_rhel8: 1.6*
     version_rhel9: 1.6*
+  v1.31.1:
+    version_rhel: 1.6*
+    version_rhel8: 1.6*
+    version_rhel9: 1.6*
 haproxy:
   version_rhel: 1.8*
   version_rhel8: 1.8*

kubemarine/resources/configurations/compatibility/internal/plugins.yaml

@@ -45,6 +45,8 @@ calico:
     version: v3.27.3
   v1.30.3:
     version: v3.28.1
+  v1.31.1:
+    version: v3.28.1
 nginx-ingress-controller:
   v1.26.3:
     version: v1.4.0
@@ -106,6 +108,9 @@ nginx-ingress-controller:
   v1.30.3:
     version: v1.11.1
     webhook-version: v1.4.1
+  v1.31.1:
+    version: v1.11.1
+    webhook-version: v1.4.1
 kubernetes-dashboard:
   v1.26.3:
     version: v2.7.0
@@ -167,6 +172,9 @@ kubernetes-dashboard:
   v1.30.3:
     version: v2.7.0
     metrics-scraper-version: v1.0.8
+  v1.31.1:
+    version: v2.7.0
+    metrics-scraper-version: v1.0.8
 local-path-provisioner:
   v1.26.3:
     version: v0.0.23
@@ -228,3 +236,6 @@ local-path-provisioner:
   v1.30.3:
     version: v0.0.27
     busybox-version: 1.34.1
+  v1.31.1:
+    version: v0.0.27
+    busybox-version: 1.34.1

kubemarine/resources/configurations/compatibility/internal/thirdparties.yaml

@@ -42,6 +42,8 @@ kubeadm:
     sha1: 85385f4b78b79bd1cd4d1ad5244422d372d5f758
   v1.30.3:
     sha1: f840e75f5dc1001ebdd7e286c0e87e1090df011b
+  v1.31.1:
+    sha1: 0a682af6436ce4e7188f93ddeebff5c2f3be1592
 kubelet:
   v1.26.3:
     sha1: 5fe320fedaabb91d3770da19135412b7454bb28b
@@ -83,6 +85,8 @@ kubelet:
     sha1: c62da6ab918b8e56d7c9b77e642ffc73ffdbffac
   v1.30.3:
     sha1: fbae53efc43ec715a45b05415294ab991ea087a2
+  v1.31.1:
+    sha1: fc7d0a9859c97ec2a2a4ac9ec1814b131e8d875f
 kubectl:
   v1.26.3:
     sha1: 56916d87c3caef05489db932fd9e48d32ebdf634
@@ -124,6 +128,8 @@ kubectl:
     sha1: bced94239f1dbdb04d3a661a067bf9587865b6e8
   v1.30.3:
     sha1: 097d6b02fabb284418a9c95ea81fa86fc3c85bb7
+  v1.31.1:
+    sha1: a0fd9dc942f533e2bdeaa4b2691fc408e334f922
 calicoctl:
 # calicoctl version is duplicated from kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml
 # It also corresponds to the plugin version in kubemarine/resources/configurations/compatibility/internal/plugins.yaml
@@ -187,6 +193,9 @@ calicoctl:
   v1.30.3:
     version: v3.28.1
     sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2
+  v1.31.1:
+    version: v3.28.1
+    sha1: 80f164a8248b8fa501f35f12cbcac0b059e665e2
 crictl:
 # crictl version is duplicated from kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml
 # for backward compatibility with clusters in a private environment.
@@ -250,3 +259,6 @@ crictl:
   v1.30.3:
     version: v1.30.0
     sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b
+  v1.31.1:
+    version: v1.30.0
+    sha1: c81e76d5d4bf64d6b513485490722d2fc0a9a83b

kubemarine/resources/configurations/compatibility/kubernetes_versions.yaml

@@ -10,6 +10,8 @@ kubernetes_versions:
     supported: true
   v1.30:
     supported: true
+  v1.31:
+    supported: true
 compatibility_map:
 # This section should be changed manually.
   v1.26.3:
@@ -132,7 +134,12 @@ compatibility_map:
     kubernetes-dashboard: v2.7.0
     local-path-provisioner: v0.0.27
     crictl: v1.30.0
-
+  v1.31.1:
+    calico: v3.28.1
+    nginx-ingress-controller: v1.11.1
+    kubernetes-dashboard: v2.7.0
+    local-path-provisioner: v0.0.27
+    crictl: v1.30.0
 # After any change, please run scripts/thirdparties/sync.py
 
 # The following optional keys are supported in addition to the 5 required software keys:

kubemarine/resources/configurations/globals.yaml

@@ -128,19 +128,19 @@ thirdparties:
   /usr/bin/kubeadm:
     software_name: kubeadm
     source_prefix:
-      public: https://storage.googleapis.com/kubernetes-release/release
+      public: https://dl.k8s.io
       private: '{registry}/kubernetes/kubeadm'
     relative_path: '{version}/bin/linux/amd64/kubeadm'
   /usr/bin/kubelet:
     software_name: kubelet
     source_prefix:
-      public: https://storage.googleapis.com/kubernetes-release/release
+      public: https://dl.k8s.io
       private: '{registry}/kubernetes/kubelet'
     relative_path: '{version}/bin/linux/amd64/kubelet'
   /usr/bin/kubectl:
     software_name: kubectl
     source_prefix:
-      public: https://storage.googleapis.com/kubernetes-release/release
+      public: https://dl.k8s.io
       private: '{registry}/kubernetes/kubectl'
     relative_path: '{version}/bin/linux/amd64/kubectl'
   /usr/bin/calicoctl:

kubemarine/resources/schemas/definitions/services/kubeadm.json

@@ -48,7 +48,10 @@
       "$ref": "#/definitions/Etcd"
     },
     "apiVersion": {"type": ["string"], "default": "kubeadm.k8s.io/v1beta2"},
-    "kind": {"enum": ["ClusterConfiguration"], "default": "ClusterConfiguration"}
+    "kind": {"enum": ["ClusterConfiguration"], "default": "ClusterConfiguration"},
+    "featureGates": {
+      "$ref": "#/definitions/FeatureGates"
+    }
   },
   "definitions": {
     "ApiServer": {
@@ -172,6 +175,15 @@
       },
       "required": ["name", "hostPath", "mountPath"],
       "additionalProperties": false
+    },
+    "FeatureGates": {
+      "type": "object",
+      "properties": {
+        "ControlPlaneKubeletLocalMode": {
+          "type": "boolean"
+        }
+      },
+      "additionalProperties": true
     }
   }
 }

test/unit/core/test_env.py

@@ -156,7 +156,7 @@ def test_kubernetes_version_env_variable(self):
 
         inventory = self.resources.working_inventory
         self.assertEqual(kubernetes_version, inventory['services']['kubeadm']['kubernetesVersion'])
-        expected_source = (f'https://storage.googleapis.com/kubernetes-release/release/'
+        expected_source = (f'https://dl.k8s.io/'
                            f'{kubernetes_version}/bin/linux/amd64/kubeadm')
         self.assertEqual(expected_source,
                          inventory['services']['thirdparties']['/usr/bin/kubeadm']['source'])
@@ -183,7 +183,7 @@ def test_kubernetes_version_upgrade_env_variable(self):
 
         inventory = self.resources.working_inventory
         self.assertEqual(after, inventory['services']['kubeadm']['kubernetesVersion'])
-        self.assertEqual(f'https://storage.googleapis.com/kubernetes-release/release/{after}/bin/linux/amd64/kubeadm',
+        self.assertEqual(f'https://dl.k8s.io/{after}/bin/linux/amd64/kubeadm',
                          inventory['services']['thirdparties']['/usr/bin/kubeadm']['source'])
         self.assertEqual('containerd_new',
                          inventory['services']['packages']['associations']['rhel']['containerd']['package_name'])

test/unit/test_kubernetes_components.py

@@ -123,6 +123,40 @@ def _test_merge_with_inventory(self, config: dict):
         self.assertEqual(True, config.get('nested', {}).get('untouched'))
         self.assertEqual([2], config.get('array'))
 
+    def test_kubelet_local_mode_enrichment(self):
+        # No featureGates for kubernetes <= 1.30
+        inventory = demo.generate_inventory(**demo.ALLINONE)
+        inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] =  'v1.30.3'
+        cluster = demo.new_cluster(inventory)
+        kubeadm = cluster.inventory['services']['kubeadm']
+        self.assertIsNone(kubeadm.get('featureGates'))
+
+        # Enriched featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+
+        inventory = demo.generate_inventory(**demo.ALLINONE)
+        inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.1'
+        cluster = demo.new_cluster(inventory)
+        kubeadm = cluster.inventory['services']['kubeadm']
+        self.assertIsNotNone(kubeadm.get('featureGates'))
+        self.assertTrue(kubeadm['featureGates'].get('ControlPlaneKubeletLocalMode'))
+
+        # Enriched featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ with not empty featureGates
+        inventory = demo.generate_inventory(**demo.ALLINONE)
+        inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.1'
+        inventory['services'].setdefault('kubeadm', {}).setdefault('featureGates', {})['foo'] = 'bar'
+        cluster = demo.new_cluster(inventory)
+        kubeadm = cluster.inventory['services']['kubeadm']
+        self.assertIsNotNone(kubeadm.get('featureGates'))
+        self.assertEqual('bar', kubeadm['featureGates'].get('foo'))
+        self.assertTrue(kubeadm['featureGates'].get('ControlPlaneKubeletLocalMode'))
+
+        # Do not change featureGates.ControlPlaneKubeletLocalMode=true for kubernetes 1.31+ if value is overridden
+        inventory = demo.generate_inventory(**demo.ALLINONE)
+        inventory['services'].setdefault('kubeadm', {})['kubernetesVersion'] = 'v1.31.1'
+        inventory['services'].setdefault('kubeadm', {}).setdefault('featureGates', {})['ControlPlaneKubeletLocalMode'] = False
+        cluster = demo.new_cluster(inventory)
+        kubeadm = cluster.inventory['services']['kubeadm']
+        self.assertIsNotNone(kubeadm.get('featureGates'))
+        self.assertFalse(kubeadm['featureGates'].get('ControlPlaneKubeletLocalMode'))
 
 class WaitForPodsTest(unittest.TestCase):
     def setUp(self):