diff --git a/.golangci.yml b/.golangci.yml index 620fc67a20..8b871b69ba 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -203,6 +203,10 @@ issues: - revive text: exported (.+) (.+) should have comment (.*)or be unexported path: "^(test/|packaging/|pkg/.*/fake/|pkg/util/testutil).*.go" + - linters: + - revive + text: a blank import should be only in a main or test package, or have a comment justifying it + path: "^packaging/.*.go" # Disable unparam "always receives" which might not be really # useful when building libraries. - linters: diff --git a/packaging/flavorgen/flavors/clusterclass_generators.go b/packaging/flavorgen/flavors/clusterclass_generators.go index 4c8d79cc47..d562272041 100644 --- a/packaging/flavorgen/flavors/clusterclass_generators.go +++ b/packaging/flavorgen/flavors/clusterclass_generators.go @@ -29,6 +29,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" ) @@ -108,10 +109,10 @@ func getWorkersClass() clusterv1.WorkersClass { func getClusterClassPatches() []clusterv1.ClusterClassPatch { return []clusterv1.ClusterClassPatch{ - createFilesArrayPatch(), + createEmptyArraysPatch(), enableSSHPatch(), infraClusterPatch(), - kubeVipEnabledPatch(), + kubevip.TopologyPatch(), } } diff --git a/packaging/flavorgen/flavors/flavors.go b/packaging/flavorgen/flavors/flavors.go index b0d67476b6..14a0d377cd 100644 --- a/packaging/flavorgen/flavors/flavors.go +++ b/packaging/flavorgen/flavors/flavors.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/crs" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip" ) const ( @@ -82,7 +83,9 @@ func MultiNodeTemplateWithKubeVIP() ([]runtime.Object, error) { vsphereCluster := newVSphereCluster() cpMachineTemplate := newVSphereMachineTemplate(env.ClusterNameVar) workerMachineTemplate := newVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar)) - controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles()) + controlPlane := newKubeadmControlplane(cpMachineTemplate, nil) + kubevip.PatchControlPlane(&controlPlane) + kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true) cluster := newCluster(vsphereCluster, &controlPlane) machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate) @@ -91,6 +94,7 @@ func MultiNodeTemplateWithKubeVIP() ([]runtime.Object, error) { if err != nil { return nil, err } + crsResourcesCPI := crs.CreateCrsResourceObjectsCPI(&clusterResourceSet) identitySecret := newIdentitySecret() @@ -149,14 +153,25 @@ func MultiNodeTemplateWithKubeVIPIgnition() ([]runtime.Object, error) { vsphereCluster := newVSphereCluster() machineTemplate := newVSphereMachineTemplate(env.ClusterNameVar) - files := newKubeVIPFiles() + controlPlane := newIgnitionKubeadmControlplane(machineTemplate, nil) + kubevip.PatchControlPlane(&controlPlane) + // CABPK requires specifying file permissions in Ignition mode. Set a default value if not set. - for i := range files { - if files[i].Permissions == "" { - files[i].Permissions = "0400" + for i := range controlPlane.Spec.KubeadmConfigSpec.Files { + if controlPlane.Spec.KubeadmConfigSpec.Files[i].Permissions == "" { + controlPlane.Spec.KubeadmConfigSpec.Files[i].Permissions = "0400" } } - controlPlane := newIgnitionKubeadmControlplane(machineTemplate, files) + + // pre and post-kubeadm commands for kube-vip workaround + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PreKubeadmCommands, + "/etc/kube-vip-prepare.sh", + ) + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands = append( + controlPlane.Spec.KubeadmConfigSpec.PostKubeadmCommands, + "/etc/kube-vip-cleanup.sh", + ) kubeadmJoinTemplate := newIgnitionKubeadmConfigTemplate() cluster := newCluster(vsphereCluster, &controlPlane) @@ -190,7 +205,9 @@ func MultiNodeTemplateWithKubeVIPNodeIPAM() ([]runtime.Object, error) { vsphereCluster := newVSphereCluster() cpMachineTemplate := newNodeIPAMVSphereMachineTemplate(env.ClusterNameVar) workerMachineTemplate := newNodeIPAMVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar)) - controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles()) + controlPlane := newKubeadmControlplane(cpMachineTemplate, nil) + kubevip.PatchControlPlane(&controlPlane) + kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true) cluster := newCluster(vsphereCluster, &controlPlane) machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate) diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go index f415b9a7ab..bb132f75c7 100644 --- a/packaging/flavorgen/flavors/generators.go +++ b/packaging/flavorgen/flavors/generators.go @@ -29,10 +29,10 @@ import ( bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" - "sigs.k8s.io/yaml" infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/identity" ) @@ -147,10 +147,10 @@ func clusterTopologyVariables() ([]clusterv1.ClusterVariable, error) { if err != nil { return nil, errors.Wrapf(err, "failed to json-encode variable ClusterNameVar: %q", env.ClusterNameVar) } - kubeVipPodYaml := kubeVIPPodYaml() - kubeVipPod, err := json.Marshal(kubeVipPodYaml) + + kubeVipVariable, err := kubevip.TopologyVariable() if err != nil { - return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) + return nil, err } infraServerValue, err := getInfraServerValue() if err != nil { @@ -169,13 +169,7 @@ func clusterTopologyVariables() ([]clusterv1.ClusterVariable, error) { Raw: infraServerValue, }, }, - { - Name: "kubeVipPodManifest", - Value: apiextensionsv1.JSON{ - - Raw: kubeVipPod, - }, - }, + *kubeVipVariable, { Name: "controlPlaneIpAddr", Value: apiextensionsv1.JSON{ @@ -556,142 +550,6 @@ func flatcarPreKubeadmCommands() []string { } } -func kubeVIPPodSpec() *corev1.Pod { - hostPathType := corev1.HostPathFileOrCreate - pod := &corev1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: util.TypeToKind(&corev1.Pod{}), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-vip", - Namespace: "kube-system", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "kube-vip", - Image: "ghcr.io/kube-vip/kube-vip:v0.6.3", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "manager", - }, - Env: []corev1.EnvVar{ - { - // Enables kube-vip control-plane functionality - Name: "cp_enable", - Value: "true", - }, - { - // Interface that the vip should bind to - Name: "vip_interface", - Value: env.VipNetworkInterfaceVar, - }, - { - // VIP IP address - // 'vip_address' was replaced by 'address' - Name: "address", - Value: env.ControlPlaneEndpointVar, - }, - { - // VIP TCP port - Name: "port", - Value: "6443", - }, - { - // Enables ARP brodcasts from Leader (requires L2 connectivity) - Name: "vip_arp", - Value: "true", - }, - { - // Kubernetes algorithm to be used. - Name: "vip_leaderelection", - Value: "true", - }, - { - // Seconds a lease is held for - Name: "vip_leaseduration", - Value: "15", - }, - { - // Seconds a leader can attempt to renew the lease - Name: "vip_renewdeadline", - Value: "10", - }, - { - // Number of times the leader will hold the lease for - Name: "vip_retryperiod", - Value: "2", - }, - { - // Enables kube-vip to watch Services of type LoadBalancer - Name: "svc_enable", - Value: "true", - }, - { - // Enables a leadership Election for each Service, allowing them to be distributed - Name: "svc_election", - Value: "true", - }, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - "NET_RAW", - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - MountPath: "/etc/kubernetes/admin.conf", - Name: "kubeconfig", - }, - }, - }, - }, - HostNetwork: true, - HostAliases: []corev1.HostAlias{ - { - IP: "127.0.0.1", - Hostnames: []string{ - "kubernetes", - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "kubeconfig", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/kubernetes/admin.conf", - Type: &hostPathType, - }, - }, - }, - }, - }, - } - return pod -} - -// kubeVIPPodYaml converts the KubeVip pod spec to a `printable` yaml -// this is needed for the file contents of KubeadmConfig. -func kubeVIPPodYaml() string { - pod := kubeVIPPodSpec() - podYaml := util.GenerateObjectYAML(pod, []util.Replacement{}) - return podYaml -} - -func kubeVIPPod() string { - pod := kubeVIPPodSpec() - podBytes, err := yaml.Marshal(pod) - if err != nil { - panic(err) - } - return string(podBytes) -} - func newClusterResourceSet(cluster clusterv1.Cluster) addonsv1.ClusterResourceSet { crs := addonsv1.ClusterResourceSet{ TypeMeta: metav1.TypeMeta{ @@ -768,16 +626,6 @@ func newMachineDeployment(cluster clusterv1.Cluster, machineTemplate infrav1.VSp } } -func newKubeVIPFiles() []bootstrapv1.File { - return []bootstrapv1.File{ - { - Owner: "root:root", - Path: "/etc/kubernetes/manifests/kube-vip.yaml", - Content: kubeVIPPod(), - }, - } -} - func newKubeadmControlplane(infraTemplate infrav1.VSphereMachineTemplate, files []bootstrapv1.File) controlplanev1.KubeadmControlPlane { return controlplanev1.KubeadmControlPlane{ TypeMeta: metav1.TypeMeta{ diff --git a/packaging/flavorgen/flavors/kubevip/files.go b/packaging/flavorgen/flavors/kubevip/files.go new file mode 100644 index 0000000000..1d0cf2830d --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/files.go @@ -0,0 +1,62 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubevip + +import ( + _ "embed" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +var ( + // This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + + //go:embed kube-vip-prepare.sh + kubeVipPrepare string + //go:embed kube-vip-cleanup.sh + kubeVipCleanup string +) + +func newKubeVIPFiles() []bootstrapv1.File { + return []bootstrapv1.File{ + { + Owner: "root:root", + Path: "/etc/kubernetes/manifests/kube-vip.yaml", + Content: kubeVIPPod(), + }, + // This two patches are part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 + { + Owner: "root:root", + Path: "/etc/kube-vip.hosts", + Permissions: "0644", + Content: "127.0.0.1 localhost kubernetes", + }, + // This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + { + Owner: "root:root", + Path: "/etc/kube-vip-prepare.sh", + Permissions: "0700", + Content: kubeVipPrepare, + }, + { + Owner: "root:root", + Path: "/etc/kube-vip-prepare.sh", + Permissions: "0700", + Content: kubeVipCleanup, + }, + } +} diff --git a/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh b/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh new file mode 100644 index 0000000000..92e7f8020e --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Reset the workaround required for kubeadm init with kube-vip: +# xref: https://github.com/kube-vip/kube-vip/issues/684 + +sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true diff --git a/packaging/flavorgen/flavors/kubevip/kube-vip-prepare.sh b/packaging/flavorgen/flavors/kubevip/kube-vip-prepare.sh new file mode 100644 index 0000000000..4cfd4e9408 --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/kube-vip-prepare.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Configure the workaround required for kubeadm init with kube-vip: +# xref: https://github.com/kube-vip/kube-vip/issues/684 + +# Nothing to do for kubernetes < v1.29 +KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" +if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return +fi + +IS_KUBEADM_INIT="false" + +# cloud-init kubeadm init +if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" +fi + +# ignition kubeadm init +if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" +fi + +if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml +fi diff --git a/packaging/flavorgen/flavors/kubevip/kubevip.go b/packaging/flavorgen/flavors/kubevip/kubevip.go new file mode 100644 index 0000000000..78f5f7660d --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/kubevip.go @@ -0,0 +1,183 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubevip exposes functions to add kubevip to templates. +package kubevip + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" +) + +var ( + hostPathTypeFile = corev1.HostPathFile + + kubeVipPodSpec = &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: util.TypeToKind(&corev1.Pod{}), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-vip", + Namespace: "kube-system", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-vip", + Image: "ghcr.io/kube-vip/kube-vip:v0.6.3", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "manager", + }, + Env: []corev1.EnvVar{ + { + // Enables kube-vip control-plane functionality + Name: "cp_enable", + Value: "true", + }, + { + // Interface that the vip should bind to + Name: "vip_interface", + Value: env.VipNetworkInterfaceVar, + }, + { + // VIP IP address + // 'vip_address' was replaced by 'address' + Name: "address", + Value: env.ControlPlaneEndpointVar, + }, + { + // VIP TCP port + Name: "port", + Value: "6443", + }, + { + // Enables ARP brodcasts from Leader (requires L2 connectivity) + Name: "vip_arp", + Value: "true", + }, + { + // Kubernetes algorithm to be used. + Name: "vip_leaderelection", + Value: "true", + }, + { + // Seconds a lease is held for + Name: "vip_leaseduration", + Value: "15", + }, + { + // Seconds a leader can attempt to renew the lease + Name: "vip_renewdeadline", + Value: "10", + }, + { + // Number of times the leader will hold the lease for + Name: "vip_retryperiod", + Value: "2", + }, + { + // Enables kube-vip to watch Services of type LoadBalancer + Name: "svc_enable", + Value: "true", + }, + { + // Enables a leadership Election for each Service, allowing them to be distributed + Name: "svc_election", + Value: "true", + }, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "NET_ADMIN", + "NET_RAW", + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/etc/kubernetes/admin.conf", + Name: "kubeconfig", + }, + // This mount is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 + { + MountPath: "/etc/hosts", + Name: "etchosts", + }, + }, + }, + }, + HostNetwork: true, + Volumes: []corev1.Volume{ + { + Name: "kubeconfig", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kubernetes/admin.conf", + Type: &hostPathTypeFile, + }, + }, + }, + // This mount is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 + { + Name: "etchosts", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kube-vip.hosts", + Type: &hostPathTypeFile, + }, + }, + }, + }, + }, + } +) + +func PatchControlPlane(cp *controlplanev1.KubeadmControlPlane) { + cp.Spec.KubeadmConfigSpec.Files = append(cp.Spec.KubeadmConfigSpec.Files, newKubeVIPFiles()...) + + // This two commands are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + cp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + cp.Spec.KubeadmConfigSpec.PreKubeadmCommands, + "/etc/kube-vip-prepare.sh", + ) + cp.Spec.KubeadmConfigSpec.PostKubeadmCommands = append( + cp.Spec.KubeadmConfigSpec.PostKubeadmCommands, + "/etc/kube-vip-cleanup.sh", + ) +} + +// kubeVIPPodYaml converts the KubeVip pod spec to a `printable` yaml +// this is needed for the file contents of KubeadmConfig. +func kubeVIPPodYaml() string { + podYaml := util.GenerateObjectYAML(kubeVipPodSpec, []util.Replacement{}) + return podYaml +} + +func kubeVIPPod() string { + podBytes, err := yaml.Marshal(kubeVipPodSpec) + if err != nil { + panic(err) + } + return string(podBytes) +} diff --git a/packaging/flavorgen/flavors/kubevip/topology.go b/packaging/flavorgen/flavors/kubevip/topology.go new file mode 100644 index 0000000000..06099eef57 --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/topology.go @@ -0,0 +1,122 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubevip + +import ( + _ "embed" + "encoding/json" + + "github.com/pkg/errors" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" +) + +// TopologyVariable returns the ClusterClass variable for kube-vip. +func TopologyVariable() (*clusterv1.ClusterVariable, error) { + kubeVipPodYaml := kubeVIPPodYaml() + kubeVipPod, err := json.Marshal(kubeVipPodYaml) + if err != nil { + return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) + } + + return &clusterv1.ClusterVariable{ + Name: "kubeVipPodManifest", + Value: apiextensionsv1.JSON{ + Raw: kubeVipPod, + }, + }, nil +} + +// TopologyKubeVipPod returns the ClusterClass patch for kube-vip. +func TopologyKubeVipPod() ([]byte, error) { + kubeVipPodYaml := kubeVIPPodYaml() + kubeVipPod, err := json.Marshal(kubeVipPodYaml) + if err != nil { + return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) + } + + return kubeVipPod, nil +} + +// TopologyPatch returns the ClusterClass patch for kube-vip. +func TopologyPatch() clusterv1.ClusterClassPatch { + patches := []clusterv1.JSONPatch{} + + for _, f := range newKubeVIPFiles() { + if f.Path == "/etc/kubernetes/manifests/kube-vip.yaml" { + f.Content = `{{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }}` + } + + tpl, _ := fileToTemplate(f) + + patches = append( + patches, + clusterv1.JSONPatch{ + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files/-", + ValueFrom: &clusterv1.JSONPatchValue{ + Template: pointer.String(tpl), + }, + }, + ) + } + + // This two patches are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + patches = append(patches, + clusterv1.JSONPatch{ + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-", + ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-prepare.sh")}, + }, + clusterv1.JSONPatch{ + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-", + ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-cleanup.sh")}, + }, + ) + + return clusterv1.ClusterClassPatch{ + Name: "kubeVipPodManifest", + Definitions: []clusterv1.PatchDefinition{ + { + Selector: clusterv1.PatchSelector{ + APIVersion: controlplanev1.GroupVersion.String(), + Kind: util.TypeToKind(&controlplanev1.KubeadmControlPlaneTemplate{}), + MatchResources: clusterv1.PatchSelectorMatch{ + ControlPlane: true, + }, + }, + JSONPatches: patches, + }, + }, + } +} + +func fileToTemplate(f bootstrapv1.File) (string, error) { + out, err := yaml.Marshal(f) + if err != nil { + return "", errors.Wrapf(err, "unable to wrap file %q", f.Path) + } + + return string(out), nil +} diff --git a/packaging/flavorgen/flavors/patches.go b/packaging/flavorgen/flavors/patches.go index 04669d7966..5ed249feb6 100644 --- a/packaging/flavorgen/flavors/patches.go +++ b/packaging/flavorgen/flavors/patches.go @@ -30,9 +30,9 @@ import ( "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" ) -func createFilesArrayPatch() clusterv1.ClusterClassPatch { +func createEmptyArraysPatch() clusterv1.ClusterClassPatch { return clusterv1.ClusterClassPatch{ - Name: "createFilesArray", + Name: "createEmptyArrays", Definitions: []clusterv1.PatchDefinition{ { Selector: clusterv1.PatchSelector{ @@ -50,6 +50,13 @@ func createFilesArrayPatch() clusterv1.ClusterClassPatch { Raw: []byte("[]"), }, }, + { + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands", + Value: &apiextensionsv1.JSON{ + Raw: []byte("[]"), + }, + }, }, }, { @@ -70,6 +77,13 @@ func createFilesArrayPatch() clusterv1.ClusterClassPatch { Raw: []byte("[]"), }, }, + { + Op: "add", + Path: "/spec/template/spec/postKubeadmCommands", + Value: &apiextensionsv1.JSON{ + Raw: []byte("[]"), + }, + }, }, }, }, @@ -170,33 +184,3 @@ func infraClusterPatch() clusterv1.ClusterClassPatch { }, } } - -func kubeVipEnabledPatch() clusterv1.ClusterClassPatch { - return clusterv1.ClusterClassPatch{ - Name: "kubeVipPodManifest", - Definitions: []clusterv1.PatchDefinition{ - { - Selector: clusterv1.PatchSelector{ - APIVersion: controlplanev1.GroupVersion.String(), - Kind: util.TypeToKind(&controlplanev1.KubeadmControlPlaneTemplate{}), - MatchResources: clusterv1.PatchSelectorMatch{ - ControlPlane: true, - }, - }, - JSONPatches: []clusterv1.JSONPatch{ - { - Op: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/files/-", - ValueFrom: &clusterv1.JSONPatchValue{ - // This patch ensures that the ControlPlaneIP which is set as variable `controlPlaneIPAddr` is also set - // in the kube-vip static pod manifest. - Template: pointer.String(`owner: root:root -path: "/etc/kubernetes/manifests/kube-vip.yaml" -content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }}`), - }, - }, - }, - }, - }, - } -} diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index dbf4b0ad9d..080c2101c1 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -123,20 +123,100 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml permissions: "0400" + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" format: ignition ignition: containerLinuxConfig: @@ -210,9 +290,14 @@ spec: kubeletExtraArgs: cloud-provider: external name: $${COREOS_CUSTOM_HOSTNAME} + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + - /etc/kube-vip-prepare.sh + - /etc/kube-vip-prepare.sh users: - name: core sshAuthorizedKeys: diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml index 27e285ee83..b550bd48b5 100644 --- a/templates/cluster-template-node-ipam.yaml +++ b/templates/cluster-template-node-ipam.yaml @@ -160,19 +160,99 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -185,12 +265,15 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ local_hostname }}' + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - /etc/kube-vip-prepare.sh users: - name: capv sshAuthorizedKeys: diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index c6bbc00370..54c1fc98be 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -64,16 +64,18 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts - name: controlPlaneIpAddr value: ${CONTROL_PLANE_ENDPOINT_IP} - name: credsSecretName diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index eca335c36b..a27f04d64c 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -150,19 +150,99 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -175,12 +255,15 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ local_hostname }}' + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - /etc/kube-vip-prepare.sh users: - name: capv sshAuthorizedKeys: diff --git a/templates/clusterclass-template.yaml b/templates/clusterclass-template.yaml index b1413f9013..b7533f7897 100644 --- a/templates/clusterclass-template.yaml +++ b/templates/clusterclass-template.yaml @@ -37,6 +37,9 @@ spec: - op: add path: /spec/template/spec/kubeadmConfigSpec/files value: [] + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands + value: [] selector: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate @@ -46,6 +49,9 @@ spec: - op: add path: /spec/template/spec/files value: [] + - op: add + path: /spec/template/spec/postKubeadmCommands + value: [] selector: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate @@ -53,7 +59,7 @@ spec: machineDeploymentClass: names: - ${CLUSTER_CLASS_NAME}-worker - name: createFilesArray + name: createEmptyArrays - definitions: - jsonPatches: - op: add @@ -120,10 +126,109 @@ spec: - op: add path: /spec/template/spec/kubeadmConfigSpec/files/- valueFrom: - template: |- + template: | + content: '{{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest + (printf "$1 %s" .controlPlaneIpAddr)) }}' + owner: root:root + path: /etc/kubernetes/manifests/kube-vip.yaml + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true owner: root:root - path: "/etc/kubernetes/manifests/kube-vip.yaml" - content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }} + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - op: add + path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/- + valueFrom: + template: /etc/kube-vip-prepare.sh + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + valueFrom: + template: /etc/kube-vip-cleanup.sh selector: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate