WIP add kube-vip workaround

kubernetes-sigs · Dec 21, 2023 · ee0ea6f · ee0ea6f
1 parent 458a6d9
commit ee0ea6f
Show file tree

Hide file tree

Showing 11 changed files with 502 additions and 25 deletions.
diff --git a/packaging/flavorgen/flavors/files/embed.go b/packaging/flavorgen/flavors/files/embed.go
@@ -0,0 +1,30 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package files exposes embedded files as strings.
+package files
+
+import _ "embed"
+
+//go:embed kube-vip-prepare.sh
+
+// KubeVIPPrepare contains the kube-vip-prepare.sh script
+var KubeVipPrepare string
+
+//go:embed kube-vip-cleanup.sh
+
+// KubeVIPCleanup contains the kube-vip-cleanup.sh script
+var KubeVipCleanup string
diff --git a/packaging/flavorgen/flavors/files/kube-vip-cleanup.sh b/packaging/flavorgen/flavors/files/kube-vip-cleanup.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Reset the workaround required for kubeadm init with kube-vip:
+# xref: https://github.com/kube-vip/kube-vip/issues/684
+
+sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \
+  /etc/kubernetes/manifests/kube-vip.yaml || true
diff --git a/packaging/flavorgen/flavors/files/kube-vip-prepare.sh b/packaging/flavorgen/flavors/files/kube-vip-prepare.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Configure the workaround required for kubeadm init with kube-vip:
+# xref: https://github.com/kube-vip/kube-vip/issues/684
+
+# Nothing to do for kubernetes < v1.29
+KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)"
+if [[ "$KUBEADM_MINOR" -lt "29" ]]; then
+  return
+fi
+
+IS_KUBEADM_INIT="false"
+
+# cloud-init kubeadm init
+if [[ -f /run/kubeadm/kubeadm.yaml ]]; then
+  IS_KUBEADM_INIT="true"
+fi
+
+# ignition kubeadm init
+if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then
+  IS_KUBEADM_INIT="true"
+fi
+
+if [[ "$IS_KUBEADM_INIT" == "true" ]]; then
+  sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \
+    /etc/kubernetes/manifests/kube-vip.yaml
+fi
diff --git a/packaging/flavorgen/flavors/flavors.go b/packaging/flavorgen/flavors/flavors.go
@@ -91,6 +91,17 @@ func MultiNodeTemplateWithKubeVIP() ([]runtime.Object, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// pre and post-kubeadm commands for kube-vip workaround
+	controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands,
+		"/etc/kube-vip-prepare.sh",
+	)
+	controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands,
+		"/etc/kube-vip-cleanup.sh",
+	)
+
 	crsResourcesCPI := crs.CreateCrsResourceObjectsCPI(&clusterResourceSet)
 	identitySecret := newIdentitySecret()
 
@@ -158,6 +169,16 @@ func MultiNodeTemplateWithKubeVIPIgnition() ([]runtime.Object, error) {
 	}
 	controlPlane := newIgnitionKubeadmControlplane(machineTemplate, files)
 
+	// pre and post-kubeadm commands for kube-vip workaround
+	controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands,
+		"/etc/kube-vip-prepare.sh",
+	)
+	controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands,
+		"/etc/kube-vip-cleanup.sh",
+	)
+
 	kubeadmJoinTemplate := newIgnitionKubeadmConfigTemplate()
 	cluster := newCluster(vsphereCluster, &controlPlane)
 	machineDeployment := newMachineDeployment(cluster, machineTemplate, kubeadmJoinTemplate)
@@ -191,6 +212,17 @@ func MultiNodeTemplateWithKubeVIPNodeIPAM() ([]runtime.Object, error) {
 	cpMachineTemplate := newNodeIPAMVSphereMachineTemplate(env.ClusterNameVar)
 	workerMachineTemplate := newNodeIPAMVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar))
 	controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles())
+
+	// pre and post-kubeadm commands for kube-vip workaround
+	controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands,
+		"/etc/kube-vip-prepare.sh",
+	)
+	controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands,
+		"/etc/kube-vip-cleanup.sh",
+	)
+
 	kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true)
 	cluster := newCluster(vsphereCluster, &controlPlane)
 	machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate)

diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go
@@ -33,6 +33,7 @@ import (
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env"
+	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/files"
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/identity"
 )
@@ -647,18 +648,14 @@ func kubeVIPPodSpec() *corev1.Pod {
 							MountPath: "/etc/kubernetes/admin.conf",
 							Name:      "kubeconfig",
 						},
+						{
+							MountPath: "/etc/hosts",
+							Name:      "etchosts",
+						},
 					},
 				},
 			},
 			HostNetwork: true,
-			HostAliases: []corev1.HostAlias{
-				{
-					IP: "127.0.0.1",
-					Hostnames: []string{
-						"kubernetes",
-					},
-				},
-			},
 			Volumes: []corev1.Volume{
 				{
 					Name: "kubeconfig",
@@ -669,6 +666,15 @@ func kubeVIPPodSpec() *corev1.Pod {
 						},
 					},
 				},
+				{
+					Name: "etchosts",
+					VolumeSource: corev1.VolumeSource{
+						HostPath: &corev1.HostPathVolumeSource{
+							Path: "/etc/kube-vip.hosts",
+							Type: &hostPathType,
+						},
+					},
+				},
 			},
 		},
 	}
@@ -775,6 +781,26 @@ func newKubeVIPFiles() []bootstrapv1.File {
 			Path:    "/etc/kubernetes/manifests/kube-vip.yaml",
 			Content: kubeVIPPod(),
 		},
+		// TODO(chrischdi) Workaround for issue X
+		{
+			Owner:       "root:root",
+			Path:        "/etc/kube-vip.hosts",
+			Permissions: "0644",
+			Content:     "127.0.0.1 localhost kubernetes",
+		},
+
+		{
+			Owner:       "root:root",
+			Path:        "/etc/kube-vip-prepare.sh",
+			Permissions: "0700",
+			Content:     files.KubeVipPrepare,
+		},
+		{
+			Owner:       "root:root",
+			Path:        "/etc/kube-vip-prepare.sh",
+			Permissions: "0700",
+			Content:     files.KubeVipCleanup,
+		},
 	}
 }
 

diff --git a/packaging/flavorgen/flavors/patches.go b/packaging/flavorgen/flavors/patches.go
@@ -172,7 +172,7 @@ func infraClusterPatch() clusterv1.ClusterClassPatch {
 }
 
 func kubeVipEnabledPatch() clusterv1.ClusterClassPatch {
-	return clusterv1.ClusterClassPatch{
+	p := clusterv1.ClusterClassPatch{
 		Name: "kubeVipPodManifest",
 		Definitions: []clusterv1.PatchDefinition{
 			{
@@ -199,4 +199,44 @@ content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipP
 			},
 		},
 	}
+
+	kubeVIPPatches := []clusterv1.JSONPatch{
+		{
+			Op:        "add",
+			Path:      "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-",
+			ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-prepare.sh")},
+		},
+		{
+			Op:        "add",
+			Path:      "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-",
+			ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-cleanup.sh")},
+		},
+	}
+
+	for _, f := range newKubeVIPFiles() {
+		if f.Path == "/etc/kubernetes/manifests/kube-vip.yaml" {
+			continue
+		}
+
+		kubeVIPPatches = append(
+			kubeVIPPatches,
+			clusterv1.JSONPatch{
+				Op:   "add",
+				Path: "/spec/template/spec/kubeadmConfigSpec/files/-",
+				ValueFrom: &clusterv1.JSONPatchValue{
+					Template: pointer.String(
+						fmt.Sprintf(`owner: %s
+permissions: "%s"
+path: "%s"
+content: %q`,
+							f.Owner, f.Permissions, f.Path, f.Content,
+						)),
+				},
+			},
+		)
+	}
+
+	p.Definitions[0].JSONPatches = append(p.Definitions[0].JSONPatches, kubeVIPPatches...)
+
+	return p
 }
diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml
@@ -123,20 +123,100 @@ spec:
             volumeMounts:
             - mountPath: /etc/kubernetes/admin.conf
               name: kubeconfig
-          hostAliases:
-          - hostnames:
-            - kubernetes
-            ip: 127.0.0.1
+            - mountPath: /etc/hosts
+              name: etchosts
           hostNetwork: true
           volumes:
           - hostPath:
               path: /etc/kubernetes/admin.conf
               type: FileOrCreate
             name: kubeconfig
+          - hostPath:
+              path: /etc/kube-vip.hosts
+              type: FileOrCreate
+            name: etchosts
         status: {}
       owner: root:root
       path: /etc/kubernetes/manifests/kube-vip.yaml
       permissions: "0400"
+    - content: 127.0.0.1 localhost kubernetes
+      owner: root:root
+      path: /etc/kube-vip.hosts
+      permissions: "0644"
+    - content: |
+        #!/bin/bash
+
+        # Copyright 2020 The Kubernetes Authors.
+        #
+        # Licensed under the Apache License, Version 2.0 (the "License");
+        # you may not use this file except in compliance with the License.
+        # You may obtain a copy of the License at
+        #
+        #     http://www.apache.org/licenses/LICENSE-2.0
+        #
+        # Unless required by applicable law or agreed to in writing, software
+        # distributed under the License is distributed on an "AS IS" BASIS,
+        # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        # See the License for the specific language governing permissions and
+        # limitations under the License.
+
+        set -e
+
+        # Configure the workaround required for kubeadm init with kube-vip:
+        # xref: https://github.com/kube-vip/kube-vip/issues/684
+
+        # Nothing to do for kubernetes < v1.29
+        KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)"
+        if [[ "$KUBEADM_MINOR" -lt "29" ]]; then
+          return
+        fi
+
+        IS_KUBEADM_INIT="false"
+
+        # cloud-init kubeadm init
+        if [[ -f /run/kubeadm/kubeadm.yaml ]]; then
+          IS_KUBEADM_INIT="true"
+        fi
+
+        # ignition kubeadm init
+        if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then
+          IS_KUBEADM_INIT="true"
+        fi
+
+        if [[ "$IS_KUBEADM_INIT" == "true" ]]; then
+          sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \
+            /etc/kubernetes/manifests/kube-vip.yaml
+        fi
+      owner: root:root
+      path: /etc/kube-vip-prepare.sh
+      permissions: "0700"
+    - content: |
+        #!/bin/bash
+
+        # Copyright 2020 The Kubernetes Authors.
+        #
+        # Licensed under the Apache License, Version 2.0 (the "License");
+        # you may not use this file except in compliance with the License.
+        # You may obtain a copy of the License at
+        #
+        #     http://www.apache.org/licenses/LICENSE-2.0
+        #
+        # Unless required by applicable law or agreed to in writing, software
+        # distributed under the License is distributed on an "AS IS" BASIS,
+        # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+        # See the License for the specific language governing permissions and
+        # limitations under the License.
+
+        set -e
+
+        # Reset the workaround required for kubeadm init with kube-vip:
+        # xref: https://github.com/kube-vip/kube-vip/issues/684
+
+        sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \
+          /etc/kubernetes/manifests/kube-vip.yaml || true
+      owner: root:root
+      path: /etc/kube-vip-prepare.sh
+      permissions: "0700"
     format: ignition
     ignition:
       containerLinuxConfig:
@@ -210,9 +290,12 @@ spec:
         kubeletExtraArgs:
           cloud-provider: external
         name: $${COREOS_CUSTOM_HOSTNAME}
+    postKubeadmCommands:
+    - /etc/kube-vip-cleanup.sh
     preKubeadmCommands:
     - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp
     - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml
+    - /etc/kube-vip-prepare.sh
     users:
     - name: core
       sshAuthorizedKeys: