WIP add kube-vip workaround

kubernetes-sigs · Dec 21, 2023 · 68dd33a · 68dd33a
1 parent 458a6d9
commit 68dd33a
Show file tree

Hide file tree

Showing 15 changed files with 867 additions and 222 deletions.
diff --git a/.golangci.yml b/.golangci.yml
@@ -203,6 +203,10 @@ issues:
     - revive
     text: exported (.+) (.+) should have comment (.*)or be unexported
     path: "^(test/|packaging/|pkg/.*/fake/|pkg/util/testutil).*.go"
+  - linters:
+    - revive
+    text: a blank import should be only in a main or test package, or have a comment justifying it
+    path: "^packaging/.*.go"
   # Disable unparam "always receives" which might not be really
   # useful when building libraries.
   - linters:

diff --git a/packaging/flavorgen/flavors/clusterclass_generators.go b/packaging/flavorgen/flavors/clusterclass_generators.go
@@ -29,6 +29,7 @@ import (
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env"
+	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip"
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util"
 )
 
@@ -108,10 +109,10 @@ func getWorkersClass() clusterv1.WorkersClass {
 
 func getClusterClassPatches() []clusterv1.ClusterClassPatch {
 	return []clusterv1.ClusterClassPatch{
-		createFilesArrayPatch(),
+		createEmptyArraysPatch(),
 		enableSSHPatch(),
 		infraClusterPatch(),
-		kubeVipEnabledPatch(),
+		kubevip.TopologyPatch(),
 	}
 }
 

diff --git a/packaging/flavorgen/flavors/flavors.go b/packaging/flavorgen/flavors/flavors.go
@@ -24,6 +24,7 @@ import (
 
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/crs"
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env"
+	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip"
 )
 
 const (
@@ -82,7 +83,9 @@ func MultiNodeTemplateWithKubeVIP() ([]runtime.Object, error) {
 	vsphereCluster := newVSphereCluster()
 	cpMachineTemplate := newVSphereMachineTemplate(env.ClusterNameVar)
 	workerMachineTemplate := newVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar))
-	controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles())
+	controlPlane := newKubeadmControlplane(cpMachineTemplate, nil)
+	kubevip.PatchControlPlane(&controlPlane)
+
 	kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true)
 	cluster := newCluster(vsphereCluster, &controlPlane)
 	machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate)
@@ -91,6 +94,7 @@ func MultiNodeTemplateWithKubeVIP() ([]runtime.Object, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	crsResourcesCPI := crs.CreateCrsResourceObjectsCPI(&clusterResourceSet)
 	identitySecret := newIdentitySecret()
 
@@ -149,14 +153,25 @@ func MultiNodeTemplateWithKubeVIPIgnition() ([]runtime.Object, error) {
 	vsphereCluster := newVSphereCluster()
 	machineTemplate := newVSphereMachineTemplate(env.ClusterNameVar)
 
-	files := newKubeVIPFiles()
+	controlPlane := newIgnitionKubeadmControlplane(machineTemplate, nil)
+	kubevip.PatchControlPlane(&controlPlane)
+
 	// CABPK requires specifying file permissions in Ignition mode. Set a default value if not set.
-	for i := range files {
-		if files[i].Permissions == "" {
-			files[i].Permissions = "0400"
+	for i := range controlPlane.Spec.KubeadmConfigSpec.Files {
+		if controlPlane.Spec.KubeadmConfigSpec.Files[i].Permissions == "" {
+			controlPlane.Spec.KubeadmConfigSpec.Files[i].Permissions = "0400"
 		}
 	}
-	controlPlane := newIgnitionKubeadmControlplane(machineTemplate, files)
+
+	// pre and post-kubeadm commands for kube-vip workaround
+	controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands,
+		"/etc/kube-vip-prepare.sh",
+	)
+	controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append(
+		controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands,
+		"/etc/kube-vip-cleanup.sh",
+	)
 
 	kubeadmJoinTemplate := newIgnitionKubeadmConfigTemplate()
 	cluster := newCluster(vsphereCluster, &controlPlane)
@@ -190,7 +205,9 @@ func MultiNodeTemplateWithKubeVIPNodeIPAM() ([]runtime.Object, error) {
 	vsphereCluster := newVSphereCluster()
 	cpMachineTemplate := newNodeIPAMVSphereMachineTemplate(env.ClusterNameVar)
 	workerMachineTemplate := newNodeIPAMVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar))
-	controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles())
+	controlPlane := newKubeadmControlplane(cpMachineTemplate, nil)
+	kubevip.PatchControlPlane(&controlPlane)
+
 	kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true)
 	cluster := newCluster(vsphereCluster, &controlPlane)
 	machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate)

diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go
@@ -29,10 +29,10 @@ import (
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1"
-	"sigs.k8s.io/yaml"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env"
+	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip"
 	"sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util"
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/identity"
 )
@@ -147,10 +147,10 @@ func clusterTopologyVariables() ([]clusterv1.ClusterVariable, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to json-encode variable ClusterNameVar: %q", env.ClusterNameVar)
 	}
-	kubeVipPodYaml := kubeVIPPodYaml()
-	kubeVipPod, err := json.Marshal(kubeVipPodYaml)
+
+	kubeVipVariable, err := kubevip.TopologyVariable()
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml)
+		return nil, err
 	}
 	infraServerValue, err := getInfraServerValue()
 	if err != nil {
@@ -169,13 +169,7 @@ func clusterTopologyVariables() ([]clusterv1.ClusterVariable, error) {
 				Raw: infraServerValue,
 			},
 		},
-		{
-			Name: "kubeVipPodManifest",
-			Value: apiextensionsv1.JSON{
-
-				Raw: kubeVipPod,
-			},
-		},
+		*kubeVipVariable,
 		{
 			Name: "controlPlaneIpAddr",
 			Value: apiextensionsv1.JSON{
@@ -556,142 +550,6 @@ func flatcarPreKubeadmCommands() []string {
 	}
 }
 
-func kubeVIPPodSpec() *corev1.Pod {
-	hostPathType := corev1.HostPathFileOrCreate
-	pod := &corev1.Pod{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "v1",
-			Kind:       util.TypeToKind(&corev1.Pod{}),
-		},
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "kube-vip",
-			Namespace: "kube-system",
-		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{
-				{
-					Name:            "kube-vip",
-					Image:           "ghcr.io/kube-vip/kube-vip:v0.6.3",
-					ImagePullPolicy: corev1.PullIfNotPresent,
-					Args: []string{
-						"manager",
-					},
-					Env: []corev1.EnvVar{
-						{
-							// Enables kube-vip control-plane functionality
-							Name:  "cp_enable",
-							Value: "true",
-						},
-						{
-							// Interface that the vip should bind to
-							Name:  "vip_interface",
-							Value: env.VipNetworkInterfaceVar,
-						},
-						{
-							// VIP IP address
-							// 'vip_address' was replaced by 'address'
-							Name:  "address",
-							Value: env.ControlPlaneEndpointVar,
-						},
-						{
-							// VIP TCP port
-							Name:  "port",
-							Value: "6443",
-						},
-						{
-							// Enables ARP brodcasts from Leader (requires L2 connectivity)
-							Name:  "vip_arp",
-							Value: "true",
-						},
-						{
-							// Kubernetes algorithm to be used.
-							Name:  "vip_leaderelection",
-							Value: "true",
-						},
-						{
-							// Seconds a lease is held for
-							Name:  "vip_leaseduration",
-							Value: "15",
-						},
-						{
-							// Seconds a leader can attempt to renew the lease
-							Name:  "vip_renewdeadline",
-							Value: "10",
-						},
-						{
-							// Number of times the leader will hold the lease for
-							Name:  "vip_retryperiod",
-							Value: "2",
-						},
-						{
-							// Enables kube-vip to watch Services of type LoadBalancer
-							Name:  "svc_enable",
-							Value: "true",
-						},
-						{
-							// Enables a leadership Election for each Service, allowing them to be distributed
-							Name:  "svc_election",
-							Value: "true",
-						},
-					},
-					SecurityContext: &corev1.SecurityContext{
-						Capabilities: &corev1.Capabilities{
-							Add: []corev1.Capability{
-								"NET_ADMIN",
-								"NET_RAW",
-							},
-						},
-					},
-					VolumeMounts: []corev1.VolumeMount{
-						{
-							MountPath: "/etc/kubernetes/admin.conf",
-							Name:      "kubeconfig",
-						},
-					},
-				},
-			},
-			HostNetwork: true,
-			HostAliases: []corev1.HostAlias{
-				{
-					IP: "127.0.0.1",
-					Hostnames: []string{
-						"kubernetes",
-					},
-				},
-			},
-			Volumes: []corev1.Volume{
-				{
-					Name: "kubeconfig",
-					VolumeSource: corev1.VolumeSource{
-						HostPath: &corev1.HostPathVolumeSource{
-							Path: "/etc/kubernetes/admin.conf",
-							Type: &hostPathType,
-						},
-					},
-				},
-			},
-		},
-	}
-	return pod
-}
-
-// kubeVIPPodYaml converts the KubeVip pod spec to a `printable` yaml
-// this is needed for the file contents of KubeadmConfig.
-func kubeVIPPodYaml() string {
-	pod := kubeVIPPodSpec()
-	podYaml := util.GenerateObjectYAML(pod, []util.Replacement{})
-	return podYaml
-}
-
-func kubeVIPPod() string {
-	pod := kubeVIPPodSpec()
-	podBytes, err := yaml.Marshal(pod)
-	if err != nil {
-		panic(err)
-	}
-	return string(podBytes)
-}
-
 func newClusterResourceSet(cluster clusterv1.Cluster) addonsv1.ClusterResourceSet {
 	crs := addonsv1.ClusterResourceSet{
 		TypeMeta: metav1.TypeMeta{
@@ -768,16 +626,6 @@ func newMachineDeployment(cluster clusterv1.Cluster, machineTemplate infrav1.VSp
 	}
 }
 
-func newKubeVIPFiles() []bootstrapv1.File {
-	return []bootstrapv1.File{
-		{
-			Owner:   "root:root",
-			Path:    "/etc/kubernetes/manifests/kube-vip.yaml",
-			Content: kubeVIPPod(),
-		},
-	}
-}
-
 func newKubeadmControlplane(infraTemplate infrav1.VSphereMachineTemplate, files []bootstrapv1.File) controlplanev1.KubeadmControlPlane {
 	return controlplanev1.KubeadmControlPlane{
 		TypeMeta: metav1.TypeMeta{

diff --git a/packaging/flavorgen/flavors/kubevip/files.go b/packaging/flavorgen/flavors/kubevip/files.go
@@ -0,0 +1,62 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubevip
+
+import (
+	_ "embed"
+
+	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
+)
+
+var (
+	// This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684
+
+	//go:embed kube-vip-prepare.sh
+	kubeVipPrepare string
+	//go:embed kube-vip-cleanup.sh
+	kubeVipCleanup string
+)
+
+func newKubeVIPFiles() []bootstrapv1.File {
+	return []bootstrapv1.File{
+		{
+			Owner:   "root:root",
+			Path:    "/etc/kubernetes/manifests/kube-vip.yaml",
+			Content: kubeVIPPod(),
+		},
+		// This two patches are part of the workaround for https://github.com/kube-vip/kube-vip/issues/692
+		{
+			Owner:       "root:root",
+			Path:        "/etc/kube-vip.hosts",
+			Permissions: "0644",
+			Content:     "127.0.0.1 localhost kubernetes",
+		},
+		// This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684
+		{
+			Owner:       "root:root",
+			Path:        "/etc/kube-vip-prepare.sh",
+			Permissions: "0700",
+			Content:     kubeVipPrepare,
+		},
+		{
+			Owner:       "root:root",
+			Path:        "/etc/kube-vip-prepare.sh",
+			Permissions: "0700",
+			Content:     kubeVipCleanup,
+		},
+	}
+}
diff --git a/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh b/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Copyright 2020 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Reset the workaround required for kubeadm init with kube-vip:
+# xref: https://github.com/kube-vip/kube-vip/issues/684
+
+sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \
+  /etc/kubernetes/manifests/kube-vip.yaml || true