diff --git a/.golangci.yml b/.golangci.yml index 620fc67a20..8b871b69ba 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -203,6 +203,10 @@ issues: - revive text: exported (.+) (.+) should have comment (.*)or be unexported path: "^(test/|packaging/|pkg/.*/fake/|pkg/util/testutil).*.go" + - linters: + - revive + text: a blank import should be only in a main or test package, or have a comment justifying it + path: "^packaging/.*.go" # Disable unparam "always receives" which might not be really # useful when building libraries. - linters: diff --git a/Makefile b/Makefile index 2bd4fe4d78..5e9dc3f0e8 100644 --- a/Makefile +++ b/Makefile @@ -294,6 +294,7 @@ generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.7 v generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the main branch $(MAKE) e2e-flavors-main cp "$(RELEASE_DIR)/main/cluster-template.yaml" "$(E2E_TEMPLATE_DIR)/main/base/cluster-template.yaml" + cp "$(RELEASE_DIR)/main/cluster-template-ignition.yaml" "$(E2E_TEMPLATE_DIR)/main/base/cluster-template-ignition.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/base" > "$(E2E_TEMPLATE_DIR)/main/cluster-template.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/hw-upgrade" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-hw-upgrade.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_TEMPLATE_DIR)/main/storage-policy" > "$(E2E_TEMPLATE_DIR)/main/cluster-template-storage-policy.yaml" diff --git a/README.md b/README.md index 5f4dc8ee9e..6e37f0d1f8 100644 --- a/README.md +++ b/README.md @@ -57,13 +57,78 @@ ecosystem. Note: These OVAs are not updated for security fixes and it is recommended to always use the latest patch version for the Kubernetes version you wish to run. For production-like environments, it is highly recommended to build and use your own custom images. -| Kubernetes | Ubuntu 18.04 | Ubuntu 20.04 | Ubuntu 22.04 | Photon 3 | Flatcar Stable | -|:----------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| -| v1.24.11 | [ova](https://storage.googleapis.com/capv-templates/v1.24.11/ubuntu-1804-kube-v1.24.11.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.24.11/ubuntu-1804-kube-v1.24.11.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.24.11/ubuntu-2004-kube-v1.24.11.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.24.11/ubuntu-2004-kube-v1.24.11.ova.sha256) | N/A | [ova](https://storage.googleapis.com/capv-templates/v1.24.11/photon-3-kube-v1.24.11.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.24.11/photon-3-kube-v1.24.11.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.24.11/flatcar-stable-3374.2.5-kube-v1.24.11.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.24.11/flatcar-stable-3374.2.5-kube-v1.24.11.ova.sha256) | -| v1.25.7 | [ova](https://storage.googleapis.com/capv-templates/v1.25.7/ubuntu-1804-kube-v1.25.7.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.25.7/ubuntu-1804-kube-v1.25.7.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.25.7/ubuntu-2004-kube-v1.25.7.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.25.7/ubuntu-2004-kube-v1.25.7.ova.sha256) | N/A | [ova](https://storage.googleapis.com/capv-templates/v1.25.7/photon-3-kube-v1.25.7.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.25.7/photon-3-kube-v1.25.7.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.25.7/flatcar-stable-3374.2.5-kube-v1.25.7.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.25.7/flatcar-stable-3374.2.5-kube-v1.25.7.ova.sha256) | -| v1.26.2 | [ova](https://storage.googleapis.com/capv-templates/v1.26.2/ubuntu-1804-kube-v1.26.2.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.26.2/ubuntu-1804-kube-v1.26.2.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.26.2/ubuntu-2004-kube-v1.26.2.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.26.2/ubuntu-2004-kube-v1.26.2.ova.sha256) | N/A | [ova](https://storage.googleapis.com/capv-templates/v1.26.2/photon-3-kube-v1.26.2.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.26.2/photon-3-kube-v1.26.2.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.26.2/flatcar-stable-3374.2.5-kube-v1.26.2.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.26.2/flatcar-stable-3374.2.5-kube-v1.26.2.ova.sha256) | -| v1.27.3 | [ova](https://storage.googleapis.com/capv-templates/v1.27.3/ubuntu-1804-kube-v1.27.3.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.27.3/ubuntu-1804-kube-v1.27.3.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.27.3/ubuntu-2004-kube-v1.27.3.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.27.3/ubuntu-2004-kube-v1.27.3.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.27.3/ubuntu-2204-kube-v1.27.3.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.27.3/ubuntu-2204-kube-v1.27.3.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.27.3/photon-3-kube-v1.27.3.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.27.3/photon-3-kube-v1.27.3.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.27.3/flatcar-stable-3510.2.4-kube-v1.27.3.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.27.3/flatcar-stable-3510.2.4-kube-v1.27.3.ova.sha256) | -| v1.28.0 | [ova](https://storage.googleapis.com/capv-templates/v1.28.0/ubuntu-1804-kube-v1.28.0.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.28.0/ubuntu-1804-kube-v1.28.0.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.28.0/ubuntu-2004-kube-v1.28.0.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.28.0/ubuntu-2004-kube-v1.28.0.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.28.0/ubuntu-2204-kube-v1.28.0.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.28.0/ubuntu-2204-kube-v1.28.0.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.28.0/photon-3-kube-v1.28.0.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.28.0/photon-3-kube-v1.28.0.ova.sha256) | [ova](https://storage.googleapis.com/capv-templates/v1.28.0/flatcar-stable-3510.2.6-kube-v1.28.0.ova), [sha256](https://storage.googleapis.com/capv-templates/v1.28.0/flatcar-stable-3510.2.6-kube-v1.28.0.ova.sha256) | + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KubernetesUbuntu 18.04Ubuntu 20.04Ubuntu 22.04Photon 3Photon 5Flatcar Stable
v1.24.11ova, sha256ova, sha256N/Aova, sha256N/Aova, sha256
v1.25.7ova, sha256ova, sha256N/Aova, sha256N/Aova, sha256
v1.26.2ova, sha256ova, sha256N/Aova, sha256N/Aova, sha256
v1.27.3ova, sha256ova, sha256ova, sha256ova, sha256N/Aova, sha256
v1.28.0ova, sha256ova, sha256ova, sha256ova, sha256N/Aova, sha256
v1.29.0N/AN/Aova, sha256ova, sha256ova, sha256ova, sha256
A full list of the published machine images for CAPV may be obtained with the following command: diff --git a/packaging/flavorgen/flavors/clusterclass_generators.go b/packaging/flavorgen/flavors/clusterclass_generators.go index 4c8d79cc47..d562272041 100644 --- a/packaging/flavorgen/flavors/clusterclass_generators.go +++ b/packaging/flavorgen/flavors/clusterclass_generators.go @@ -29,6 +29,7 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" ) @@ -108,10 +109,10 @@ func getWorkersClass() clusterv1.WorkersClass { func getClusterClassPatches() []clusterv1.ClusterClassPatch { return []clusterv1.ClusterClassPatch{ - createFilesArrayPatch(), + createEmptyArraysPatch(), enableSSHPatch(), infraClusterPatch(), - kubeVipEnabledPatch(), + kubevip.TopologyPatch(), } } diff --git a/packaging/flavorgen/flavors/flavors.go b/packaging/flavorgen/flavors/flavors.go index b0d67476b6..9b1356f0f9 100644 --- a/packaging/flavorgen/flavors/flavors.go +++ b/packaging/flavorgen/flavors/flavors.go @@ -24,6 +24,7 @@ import ( "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/crs" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip" ) const ( @@ -82,7 +83,9 @@ func MultiNodeTemplateWithKubeVIP() ([]runtime.Object, error) { vsphereCluster := newVSphereCluster() cpMachineTemplate := newVSphereMachineTemplate(env.ClusterNameVar) workerMachineTemplate := newVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar)) - controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles()) + controlPlane := newKubeadmControlplane(cpMachineTemplate, nil) + kubevip.PatchControlPlane(&controlPlane) + kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true) cluster := newCluster(vsphereCluster, &controlPlane) machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate) @@ -149,14 +152,15 @@ func MultiNodeTemplateWithKubeVIPIgnition() ([]runtime.Object, error) { vsphereCluster := newVSphereCluster() machineTemplate := newVSphereMachineTemplate(env.ClusterNameVar) - files := newKubeVIPFiles() + controlPlane := newIgnitionKubeadmControlplane(machineTemplate, nil) + kubevip.PatchControlPlane(&controlPlane) + // CABPK requires specifying file permissions in Ignition mode. Set a default value if not set. - for i := range files { - if files[i].Permissions == "" { - files[i].Permissions = "0400" + for i := range controlPlane.Spec.KubeadmConfigSpec.Files { + if controlPlane.Spec.KubeadmConfigSpec.Files[i].Permissions == "" { + controlPlane.Spec.KubeadmConfigSpec.Files[i].Permissions = "0400" } } - controlPlane := newIgnitionKubeadmControlplane(machineTemplate, files) kubeadmJoinTemplate := newIgnitionKubeadmConfigTemplate() cluster := newCluster(vsphereCluster, &controlPlane) @@ -190,7 +194,9 @@ func MultiNodeTemplateWithKubeVIPNodeIPAM() ([]runtime.Object, error) { vsphereCluster := newVSphereCluster() cpMachineTemplate := newNodeIPAMVSphereMachineTemplate(env.ClusterNameVar) workerMachineTemplate := newNodeIPAMVSphereMachineTemplate(fmt.Sprintf("%s-worker", env.ClusterNameVar)) - controlPlane := newKubeadmControlplane(cpMachineTemplate, newKubeVIPFiles()) + controlPlane := newKubeadmControlplane(cpMachineTemplate, nil) + kubevip.PatchControlPlane(&controlPlane) + kubeadmJoinTemplate := newKubeadmConfigTemplate(fmt.Sprintf("%s%s", env.ClusterNameVar, env.MachineDeploymentNameSuffix), true) cluster := newCluster(vsphereCluster, &controlPlane) machineDeployment := newMachineDeployment(cluster, workerMachineTemplate, kubeadmJoinTemplate) diff --git a/packaging/flavorgen/flavors/generators.go b/packaging/flavorgen/flavors/generators.go index f415b9a7ab..bb132f75c7 100644 --- a/packaging/flavorgen/flavors/generators.go +++ b/packaging/flavorgen/flavors/generators.go @@ -29,10 +29,10 @@ import ( bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" addonsv1 "sigs.k8s.io/cluster-api/exp/addons/api/v1beta1" - "sigs.k8s.io/yaml" infrav1 "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/kubevip" "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/identity" ) @@ -147,10 +147,10 @@ func clusterTopologyVariables() ([]clusterv1.ClusterVariable, error) { if err != nil { return nil, errors.Wrapf(err, "failed to json-encode variable ClusterNameVar: %q", env.ClusterNameVar) } - kubeVipPodYaml := kubeVIPPodYaml() - kubeVipPod, err := json.Marshal(kubeVipPodYaml) + + kubeVipVariable, err := kubevip.TopologyVariable() if err != nil { - return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) + return nil, err } infraServerValue, err := getInfraServerValue() if err != nil { @@ -169,13 +169,7 @@ func clusterTopologyVariables() ([]clusterv1.ClusterVariable, error) { Raw: infraServerValue, }, }, - { - Name: "kubeVipPodManifest", - Value: apiextensionsv1.JSON{ - - Raw: kubeVipPod, - }, - }, + *kubeVipVariable, { Name: "controlPlaneIpAddr", Value: apiextensionsv1.JSON{ @@ -556,142 +550,6 @@ func flatcarPreKubeadmCommands() []string { } } -func kubeVIPPodSpec() *corev1.Pod { - hostPathType := corev1.HostPathFileOrCreate - pod := &corev1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: util.TypeToKind(&corev1.Pod{}), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-vip", - Namespace: "kube-system", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "kube-vip", - Image: "ghcr.io/kube-vip/kube-vip:v0.6.3", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "manager", - }, - Env: []corev1.EnvVar{ - { - // Enables kube-vip control-plane functionality - Name: "cp_enable", - Value: "true", - }, - { - // Interface that the vip should bind to - Name: "vip_interface", - Value: env.VipNetworkInterfaceVar, - }, - { - // VIP IP address - // 'vip_address' was replaced by 'address' - Name: "address", - Value: env.ControlPlaneEndpointVar, - }, - { - // VIP TCP port - Name: "port", - Value: "6443", - }, - { - // Enables ARP brodcasts from Leader (requires L2 connectivity) - Name: "vip_arp", - Value: "true", - }, - { - // Kubernetes algorithm to be used. - Name: "vip_leaderelection", - Value: "true", - }, - { - // Seconds a lease is held for - Name: "vip_leaseduration", - Value: "15", - }, - { - // Seconds a leader can attempt to renew the lease - Name: "vip_renewdeadline", - Value: "10", - }, - { - // Number of times the leader will hold the lease for - Name: "vip_retryperiod", - Value: "2", - }, - { - // Enables kube-vip to watch Services of type LoadBalancer - Name: "svc_enable", - Value: "true", - }, - { - // Enables a leadership Election for each Service, allowing them to be distributed - Name: "svc_election", - Value: "true", - }, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - "NET_RAW", - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - MountPath: "/etc/kubernetes/admin.conf", - Name: "kubeconfig", - }, - }, - }, - }, - HostNetwork: true, - HostAliases: []corev1.HostAlias{ - { - IP: "127.0.0.1", - Hostnames: []string{ - "kubernetes", - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "kubeconfig", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/kubernetes/admin.conf", - Type: &hostPathType, - }, - }, - }, - }, - }, - } - return pod -} - -// kubeVIPPodYaml converts the KubeVip pod spec to a `printable` yaml -// this is needed for the file contents of KubeadmConfig. -func kubeVIPPodYaml() string { - pod := kubeVIPPodSpec() - podYaml := util.GenerateObjectYAML(pod, []util.Replacement{}) - return podYaml -} - -func kubeVIPPod() string { - pod := kubeVIPPodSpec() - podBytes, err := yaml.Marshal(pod) - if err != nil { - panic(err) - } - return string(podBytes) -} - func newClusterResourceSet(cluster clusterv1.Cluster) addonsv1.ClusterResourceSet { crs := addonsv1.ClusterResourceSet{ TypeMeta: metav1.TypeMeta{ @@ -768,16 +626,6 @@ func newMachineDeployment(cluster clusterv1.Cluster, machineTemplate infrav1.VSp } } -func newKubeVIPFiles() []bootstrapv1.File { - return []bootstrapv1.File{ - { - Owner: "root:root", - Path: "/etc/kubernetes/manifests/kube-vip.yaml", - Content: kubeVIPPod(), - }, - } -} - func newKubeadmControlplane(infraTemplate infrav1.VSphereMachineTemplate, files []bootstrapv1.File) controlplanev1.KubeadmControlPlane { return controlplanev1.KubeadmControlPlane{ TypeMeta: metav1.TypeMeta{ diff --git a/packaging/flavorgen/flavors/kubevip/files.go b/packaging/flavorgen/flavors/kubevip/files.go new file mode 100644 index 0000000000..6a8a0b235b --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/files.go @@ -0,0 +1,63 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubevip + +import ( + _ "embed" + + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" +) + +var ( + // This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + + //go:embed kube-vip-prepare.sh + kubeVipPrepare string + //go:embed kube-vip-cleanup.sh + kubeVipCleanup string +) + +func newKubeVIPFiles() []bootstrapv1.File { + return []bootstrapv1.File{ + { + Owner: "root:root", + Path: "/etc/kubernetes/manifests/kube-vip.yaml", + Content: kubeVIPPod(), + Permissions: "0644", + }, + // This file is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 + { + Owner: "root:root", + Path: "/etc/kube-vip.hosts", + Permissions: "0644", + Content: "127.0.0.1 localhost kubernetes", + }, + // This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + { + Owner: "root:root", + Path: "/etc/kube-vip-prepare.sh", + Permissions: "0700", + Content: kubeVipPrepare, + }, + { + Owner: "root:root", + Path: "/etc/kube-vip-cleanup.sh", + Permissions: "0700", + Content: kubeVipCleanup, + }, + } +} diff --git a/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh b/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh new file mode 100644 index 0000000000..92e7f8020e --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/kube-vip-cleanup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Reset the workaround required for kubeadm init with kube-vip: +# xref: https://github.com/kube-vip/kube-vip/issues/684 + +sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true diff --git a/packaging/flavorgen/flavors/kubevip/kube-vip-prepare.sh b/packaging/flavorgen/flavors/kubevip/kube-vip-prepare.sh new file mode 100644 index 0000000000..4cfd4e9408 --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/kube-vip-prepare.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright 2020 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Configure the workaround required for kubeadm init with kube-vip: +# xref: https://github.com/kube-vip/kube-vip/issues/684 + +# Nothing to do for kubernetes < v1.29 +KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" +if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return +fi + +IS_KUBEADM_INIT="false" + +# cloud-init kubeadm init +if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" +fi + +# ignition kubeadm init +if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" +fi + +if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml +fi diff --git a/packaging/flavorgen/flavors/kubevip/kubevip.go b/packaging/flavorgen/flavors/kubevip/kubevip.go new file mode 100644 index 0000000000..78f5f7660d --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/kubevip.go @@ -0,0 +1,183 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubevip exposes functions to add kubevip to templates. +package kubevip + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" +) + +var ( + hostPathTypeFile = corev1.HostPathFile + + kubeVipPodSpec = &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: util.TypeToKind(&corev1.Pod{}), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-vip", + Namespace: "kube-system", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "kube-vip", + Image: "ghcr.io/kube-vip/kube-vip:v0.6.3", + ImagePullPolicy: corev1.PullIfNotPresent, + Args: []string{ + "manager", + }, + Env: []corev1.EnvVar{ + { + // Enables kube-vip control-plane functionality + Name: "cp_enable", + Value: "true", + }, + { + // Interface that the vip should bind to + Name: "vip_interface", + Value: env.VipNetworkInterfaceVar, + }, + { + // VIP IP address + // 'vip_address' was replaced by 'address' + Name: "address", + Value: env.ControlPlaneEndpointVar, + }, + { + // VIP TCP port + Name: "port", + Value: "6443", + }, + { + // Enables ARP brodcasts from Leader (requires L2 connectivity) + Name: "vip_arp", + Value: "true", + }, + { + // Kubernetes algorithm to be used. + Name: "vip_leaderelection", + Value: "true", + }, + { + // Seconds a lease is held for + Name: "vip_leaseduration", + Value: "15", + }, + { + // Seconds a leader can attempt to renew the lease + Name: "vip_renewdeadline", + Value: "10", + }, + { + // Number of times the leader will hold the lease for + Name: "vip_retryperiod", + Value: "2", + }, + { + // Enables kube-vip to watch Services of type LoadBalancer + Name: "svc_enable", + Value: "true", + }, + { + // Enables a leadership Election for each Service, allowing them to be distributed + Name: "svc_election", + Value: "true", + }, + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{ + "NET_ADMIN", + "NET_RAW", + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/etc/kubernetes/admin.conf", + Name: "kubeconfig", + }, + // This mount is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 + { + MountPath: "/etc/hosts", + Name: "etchosts", + }, + }, + }, + }, + HostNetwork: true, + Volumes: []corev1.Volume{ + { + Name: "kubeconfig", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kubernetes/admin.conf", + Type: &hostPathTypeFile, + }, + }, + }, + // This mount is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 + { + Name: "etchosts", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kube-vip.hosts", + Type: &hostPathTypeFile, + }, + }, + }, + }, + }, + } +) + +func PatchControlPlane(cp *controlplanev1.KubeadmControlPlane) { + cp.Spec.KubeadmConfigSpec.Files = append(cp.Spec.KubeadmConfigSpec.Files, newKubeVIPFiles()...) + + // This two commands are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + cp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( + cp.Spec.KubeadmConfigSpec.PreKubeadmCommands, + "/etc/kube-vip-prepare.sh", + ) + cp.Spec.KubeadmConfigSpec.PostKubeadmCommands = append( + cp.Spec.KubeadmConfigSpec.PostKubeadmCommands, + "/etc/kube-vip-cleanup.sh", + ) +} + +// kubeVIPPodYaml converts the KubeVip pod spec to a `printable` yaml +// this is needed for the file contents of KubeadmConfig. +func kubeVIPPodYaml() string { + podYaml := util.GenerateObjectYAML(kubeVipPodSpec, []util.Replacement{}) + return podYaml +} + +func kubeVIPPod() string { + podBytes, err := yaml.Marshal(kubeVipPodSpec) + if err != nil { + panic(err) + } + return string(podBytes) +} diff --git a/packaging/flavorgen/flavors/kubevip/topology.go b/packaging/flavorgen/flavors/kubevip/topology.go new file mode 100644 index 0000000000..d80abc9b44 --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/topology.go @@ -0,0 +1,130 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubevip + +import ( + _ "embed" + "encoding/json" + "fmt" + "strings" + + "github.com/pkg/errors" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/utils/pointer" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/yaml" + + "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" +) + +// TopologyVariable returns the ClusterClass variable for kube-vip. +func TopologyVariable() (*clusterv1.ClusterVariable, error) { + kubeVipPodYaml := kubeVIPPodYaml() + kubeVipPod, err := json.Marshal(kubeVipPodYaml) + if err != nil { + return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) + } + + return &clusterv1.ClusterVariable{ + Name: "kubeVipPodManifest", + Value: apiextensionsv1.JSON{ + Raw: kubeVipPod, + }, + }, nil +} + +// TopologyKubeVipPod returns the ClusterClass patch for kube-vip. +func TopologyKubeVipPod() ([]byte, error) { + kubeVipPodYaml := kubeVIPPodYaml() + kubeVipPod, err := json.Marshal(kubeVipPodYaml) + if err != nil { + return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) + } + + return kubeVipPod, nil +} + +// TopologyPatch returns the ClusterClass patch for kube-vip. +func TopologyPatch() clusterv1.ClusterClassPatch { + patches := []clusterv1.JSONPatch{} + + for _, f := range newKubeVIPFiles() { + p := clusterv1.JSONPatch{ + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/files/-", + ValueFrom: &clusterv1.JSONPatchValue{}, + } + + // Special handling to make this patch work + if f.Path == "/etc/kubernetes/manifests/kube-vip.yaml" { + lines := []string{ + fmt.Sprintf("owner: %q", f.Owner), + fmt.Sprintf("path: %q", f.Path), + `content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }}`, + fmt.Sprintf("permissions: %q", f.Permissions), + } + p.ValueFrom.Template = pointer.String(strings.Join(lines, "\n")) + patches = append(patches, p) + continue + } + + tpl, _ := fileToTemplate(f) + p.ValueFrom.Template = pointer.String(tpl) + patches = append(patches, p) + } + + // This two patches are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + patches = append(patches, + clusterv1.JSONPatch{ + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-", + ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-prepare.sh")}, + }, + clusterv1.JSONPatch{ + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/-", + ValueFrom: &clusterv1.JSONPatchValue{Template: pointer.String("/etc/kube-vip-cleanup.sh")}, + }, + ) + + return clusterv1.ClusterClassPatch{ + Name: "kubeVipPodManifest", + Definitions: []clusterv1.PatchDefinition{ + { + Selector: clusterv1.PatchSelector{ + APIVersion: controlplanev1.GroupVersion.String(), + Kind: util.TypeToKind(&controlplanev1.KubeadmControlPlaneTemplate{}), + MatchResources: clusterv1.PatchSelectorMatch{ + ControlPlane: true, + }, + }, + JSONPatches: patches, + }, + }, + } +} + +func fileToTemplate(f bootstrapv1.File) (string, error) { + out, err := yaml.Marshal(f) + if err != nil { + return "", errors.Wrapf(err, "unable to wrap file %q", f.Path) + } + + return string(out), nil +} diff --git a/packaging/flavorgen/flavors/patches.go b/packaging/flavorgen/flavors/patches.go index 04669d7966..5ed249feb6 100644 --- a/packaging/flavorgen/flavors/patches.go +++ b/packaging/flavorgen/flavors/patches.go @@ -30,9 +30,9 @@ import ( "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" ) -func createFilesArrayPatch() clusterv1.ClusterClassPatch { +func createEmptyArraysPatch() clusterv1.ClusterClassPatch { return clusterv1.ClusterClassPatch{ - Name: "createFilesArray", + Name: "createEmptyArrays", Definitions: []clusterv1.PatchDefinition{ { Selector: clusterv1.PatchSelector{ @@ -50,6 +50,13 @@ func createFilesArrayPatch() clusterv1.ClusterClassPatch { Raw: []byte("[]"), }, }, + { + Op: "add", + Path: "/spec/template/spec/kubeadmConfigSpec/postKubeadmCommands", + Value: &apiextensionsv1.JSON{ + Raw: []byte("[]"), + }, + }, }, }, { @@ -70,6 +77,13 @@ func createFilesArrayPatch() clusterv1.ClusterClassPatch { Raw: []byte("[]"), }, }, + { + Op: "add", + Path: "/spec/template/spec/postKubeadmCommands", + Value: &apiextensionsv1.JSON{ + Raw: []byte("[]"), + }, + }, }, }, }, @@ -170,33 +184,3 @@ func infraClusterPatch() clusterv1.ClusterClassPatch { }, } } - -func kubeVipEnabledPatch() clusterv1.ClusterClassPatch { - return clusterv1.ClusterClassPatch{ - Name: "kubeVipPodManifest", - Definitions: []clusterv1.PatchDefinition{ - { - Selector: clusterv1.PatchSelector{ - APIVersion: controlplanev1.GroupVersion.String(), - Kind: util.TypeToKind(&controlplanev1.KubeadmControlPlaneTemplate{}), - MatchResources: clusterv1.PatchSelectorMatch{ - ControlPlane: true, - }, - }, - JSONPatches: []clusterv1.JSONPatch{ - { - Op: "add", - Path: "/spec/template/spec/kubeadmConfigSpec/files/-", - ValueFrom: &clusterv1.JSONPatchValue{ - // This patch ensures that the ControlPlaneIP which is set as variable `controlPlaneIPAddr` is also set - // in the kube-vip static pod manifest. - Template: pointer.String(`owner: root:root -path: "/etc/kubernetes/manifests/kube-vip.yaml" -content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }}`), - }, - }, - }, - }, - }, - } -} diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index dbf4b0ad9d..62ffcf6304 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -123,20 +123,100 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml - permissions: "0400" + permissions: "0644" + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-cleanup.sh + permissions: "0700" format: ignition ignition: containerLinuxConfig: @@ -210,9 +290,12 @@ spec: kubeletExtraArgs: cloud-provider: external name: $${COREOS_CUSTOM_HOSTNAME} + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml + - /etc/kube-vip-prepare.sh users: - name: core sshAuthorizedKeys: diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml index 27e285ee83..1c70734cc2 100644 --- a/templates/cluster-template-node-ipam.yaml +++ b/templates/cluster-template-node-ipam.yaml @@ -160,19 +160,100 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0644" + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-cleanup.sh + permissions: "0700" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -185,12 +266,15 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ local_hostname }}' + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - /etc/kube-vip-prepare.sh users: - name: capv sshAuthorizedKeys: diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index c6bbc00370..54c1fc98be 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -64,16 +64,18 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts - name: controlPlaneIpAddr value: ${CONTROL_PLANE_ENDPOINT_IP} - name: credsSecretName diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index eca335c36b..422942d5ae 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -150,19 +150,100 @@ spec: volumeMounts: - mountPath: /etc/kubernetes/admin.conf name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 + - mountPath: /etc/hosts + name: etchosts hostNetwork: true volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: FileOrCreate + type: File name: kubeconfig + - hostPath: + path: /etc/kube-vip.hosts + type: File + name: etchosts status: {} owner: root:root path: /etc/kubernetes/manifests/kube-vip.yaml + permissions: "0644" + - content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-cleanup.sh + permissions: "0700" initConfiguration: nodeRegistration: criSocket: /var/run/containerd/containerd.sock @@ -175,12 +256,15 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ local_hostname }}' + postKubeadmCommands: + - /etc/kube-vip-cleanup.sh preKubeadmCommands: - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" - echo "::1 ipv6-localhost ipv6-loopback localhost6 localhost6.localdomain6" >/etc/hosts - echo "127.0.0.1 {{ ds.meta_data.hostname }} {{ local_hostname }} localhost localhost.localdomain localhost4 localhost4.localdomain4" >>/etc/hosts + - /etc/kube-vip-prepare.sh users: - name: capv sshAuthorizedKeys: diff --git a/templates/clusterclass-template.yaml b/templates/clusterclass-template.yaml index b1413f9013..19b16f8f99 100644 --- a/templates/clusterclass-template.yaml +++ b/templates/clusterclass-template.yaml @@ -37,6 +37,9 @@ spec: - op: add path: /spec/template/spec/kubeadmConfigSpec/files value: [] + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands + value: [] selector: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate @@ -46,6 +49,9 @@ spec: - op: add path: /spec/template/spec/files value: [] + - op: add + path: /spec/template/spec/postKubeadmCommands + value: [] selector: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate @@ -53,7 +59,7 @@ spec: machineDeploymentClass: names: - ${CLUSTER_CLASS_NAME}-worker - name: createFilesArray + name: createEmptyArrays - definitions: - jsonPatches: - op: add @@ -121,9 +127,108 @@ spec: path: /spec/template/spec/kubeadmConfigSpec/files/- valueFrom: template: |- - owner: root:root - path: "/etc/kubernetes/manifests/kube-vip.yaml" + owner: "root:root" + path: "/etc/kubernetes/manifests/kube-vip.yaml" content: {{ printf "%q" (regexReplaceAll "(name: address\n +value:).*" .kubeVipPodManifest (printf "$1 %s" .controlPlaneIpAddr)) }} + permissions: "0644" + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + content: 127.0.0.1 localhost kubernetes + owner: root:root + path: /etc/kube-vip.hosts + permissions: "0644" + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + return + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + path: /etc/kube-vip-prepare.sh + permissions: "0700" + - op: add + path: /spec/template/spec/kubeadmConfigSpec/files/- + valueFrom: + template: | + content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Reset the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml || true + owner: root:root + path: /etc/kube-vip-cleanup.sh + permissions: "0700" + - op: add + path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/- + valueFrom: + template: /etc/kube-vip-prepare.sh + - op: add + path: /spec/template/spec/kubeadmConfigSpec/postKubeadmCommands/- + valueFrom: + template: /etc/kube-vip-cleanup.sh selector: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlaneTemplate diff --git a/test/e2e/config/vsphere-ci.yaml b/test/e2e/config/vsphere-ci.yaml index 4bc9d46230..d8715c9a66 100644 --- a/test/e2e/config/vsphere-ci.yaml +++ b/test/e2e/config/vsphere-ci.yaml @@ -178,7 +178,7 @@ providers: - sourcePath: "../data/shared/v1.7/v1beta1_provider/metadata.yaml" variables: - KUBERNETES_VERSION: "v1.28.0" + KUBERNETES_VERSION: "v1.29.0" CPI_IMAGE_K8S_VERSION: "v1.28.0" CNI: "./data/cni/calico/calico.yaml" EXP_CLUSTER_RESOURCE_SET: "true" @@ -194,8 +194,8 @@ variables: VSPHERE_DATASTORE: "WorkloadDatastore" VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy" VSPHERE_NETWORK: "sddc-cgw-network-6" - VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.28.0" - FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3510.2.6-kube-v1.28.0" + VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.29.0" + FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3602.2.3-kube-v1.29.0" VSPHERE_INSECURE_CSI: "true" KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" NODE_DRAIN_TIMEOUT: "60s" diff --git a/test/e2e/config/vsphere-dev.yaml b/test/e2e/config/vsphere-dev.yaml index 7bf2d07486..8b09bd64c0 100644 --- a/test/e2e/config/vsphere-dev.yaml +++ b/test/e2e/config/vsphere-dev.yaml @@ -181,7 +181,7 @@ providers: - sourcePath: "../data/shared/v1.7/v1beta1_provider/metadata.yaml" variables: - KUBERNETES_VERSION: "v1.28.0" + KUBERNETES_VERSION: "v1.29.0" CPI_IMAGE_K8S_VERSION: "v1.28.0" CNI: "./data/cni/calico/calico.yaml" EXP_CLUSTER_RESOURCE_SET: "true" @@ -200,8 +200,8 @@ variables: VSPHERE_DATASTORE: "WorkloadDatastore" VSPHERE_STORAGE_POLICY: "Cluster API vSphere Storage Policy" VSPHERE_NETWORK: "network-1" - VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.28.0" - FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3510.2.6-kube-v1.28.0" + VSPHERE_TEMPLATE: "ubuntu-2204-kube-v1.29.0" + FLATCAR_VSPHERE_TEMPLATE: "flatcar-stable-3602.2.3-kube-v1.29.0" # WORKLOAD_CONTROL_PLANE_ENDPOINT_IP: # Also following variables are required but it is recommended to use env variables to avoid disclosure of sensitive data # VSPHERE_SSH_AUTHORIZED_KEY: @@ -223,7 +223,7 @@ variables: VSPHERE2_SERVER: "vcenter2.vmware.com" VSPHERE2_TLS_THUMBPRINT: "AA:BB:CC:DD:11:22:33:44:EE:FF" VSPHERE2_RESOURCE_POOL: "ResourcePool" - VSPHERE2_TEMPLATE: "ubuntu-2204-kube-v1.28.0" + VSPHERE2_TEMPLATE: "ubuntu-2204-kube-v1.29.0" # Dedicated IP to be used by kube-vip VSPHERE2_CONTROL_PLANE_ENDPOINT_IP: # Following variables are also required and please use env variables to avoid disclosure of sensitive data diff --git a/test/e2e/data/infrastructure-vsphere/main/ignition/flatcar.yaml b/test/e2e/data/infrastructure-vsphere/main/ignition/flatcar.yaml new file mode 100644 index 0000000000..9b4c0b4798 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere/main/ignition/flatcar.yaml @@ -0,0 +1,9 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + template: + spec: + template: '${FLATCAR_VSPHERE_TEMPLATE}' diff --git a/test/e2e/data/infrastructure-vsphere/main/ignition/ignition.yaml b/test/e2e/data/infrastructure-vsphere/main/ignition/ignition.yaml deleted file mode 100644 index 7717a696a8..0000000000 --- a/test/e2e/data/infrastructure-vsphere/main/ignition/ignition.yaml +++ /dev/null @@ -1,254 +0,0 @@ ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: '${CLUSTER_NAME}' - namespace: '${NAMESPACE}' -spec: - kubeadmConfigSpec: - # Ignition mode in CABPK expects an explicit file mode. - # TODO: Because of https://github.com/kubernetes-sigs/cluster-api/issues/6147 we can't override - # just the permissions field and must copy the entire file element. - files: - - content: | - apiVersion: v1 - kind: Pod - metadata: - creationTimestamp: null - name: kube-vip - namespace: kube-system - spec: - containers: - - args: - - manager - env: - - name: cp_enable - value: "true" - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE:=""} - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - - name: port - value: "6443" - - name: vip_arp - value: "true" - - name: vip_leaderelection - value: "true" - - name: vip_leaseduration - value: "15" - - name: vip_renewdeadline - value: "10" - - name: vip_retryperiod - value: "2" - image: ghcr.io/kube-vip/kube-vip:v0.4.1 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - volumeMounts: - - mountPath: /etc/kubernetes/admin.conf - name: kubeconfig - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 - hostNetwork: true - volumes: - - hostPath: - path: /etc/kubernetes/admin.conf - type: FileOrCreate - name: kubeconfig - status: {} - owner: root:root - path: /etc/kubernetes/manifests/kube-vip.yaml - permissions: "0644" - format: ignition - ignition: - containerLinuxConfig: - additionalConfig: | - storage: - files: - - path: /opt/set-hostname - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/sh - set -x - echo "$${COREOS_CUSTOM_HOSTNAME}" > /etc/hostname - hostname "$${COREOS_CUSTOM_HOSTNAME}" - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - echo "127.0.0.1 localhost" >>/etc/hosts - echo "127.0.0.1 $${COREOS_CUSTOM_HOSTNAME}" >>/etc/hosts - systemd: - units: - - name: coreos-metadata.service - contents: | - [Unit] - Description=VMware metadata agent - After=nss-lookup.target - After=network-online.target - Wants=network-online.target - [Service] - Type=oneshot - Restart=on-failure - RemainAfterExit=yes - Environment=OUTPUT=/run/metadata/coreos - ExecStart=/usr/bin/mkdir --parent /run/metadata - ExecStart=/usr/bin/bash -cv 'echo "COREOS_CUSTOM_HOSTNAME=$(/usr/share/oem/bin/vmtoolsd --cmd "info-get guestinfo.metadata" | base64 -d | grep local-hostname | awk {\'print $2\'} | tr -d \'"\')" > $${OUTPUT}' - - name: set-hostname.service - enabled: true - contents: | - [Unit] - Description=Set the hostname for this machine - Requires=coreos-metadata.service - After=coreos-metadata.service - [Service] - Type=oneshot - EnvironmentFile=/run/metadata/coreos - ExecStart=/opt/set-hostname - [Install] - WantedBy=multi-user.target - - name: kubeadm.service - enabled: true - dropins: - - name: 10-flatcar.conf - contents: | - [Unit] - # kubeadm must run after coreos-metadata populated /run/metadata directory. - Requires=coreos-metadata.service - After=coreos-metadata.service - # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. - After=containerd.service - [Service] - # Make metadata environment variables available for pre-kubeadm commands. - EnvironmentFile=/run/metadata/* - initConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - cloud-provider: external - name: $${COREOS_CUSTOM_HOSTNAME} - joinConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - cloud-provider: external - name: $${COREOS_CUSTOM_HOSTNAME} - preKubeadmCommands: - - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp - - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml - useExperimentalRetryJoin: false - users: - - name: core - sshAuthorizedKeys: - - '${VSPHERE_SSH_AUTHORIZED_KEY}' - sudo: ALL=(ALL) NOPASSWD:ALL ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: '${CLUSTER_NAME}-md-0' - namespace: '${NAMESPACE}' -spec: - template: - spec: - format: ignition - ignition: - containerLinuxConfig: - additionalConfig: | - storage: - files: - - path: /opt/set-hostname - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/sh - set -x - echo "$${COREOS_CUSTOM_HOSTNAME}" > /etc/hostname - hostname "$${COREOS_CUSTOM_HOSTNAME}" - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts - echo "127.0.0.1 localhost" >>/etc/hosts - echo "127.0.0.1 $${COREOS_CUSTOM_HOSTNAME}" >>/etc/hosts - systemd: - units: - - name: coreos-metadata.service - contents: | - [Unit] - Description=VMware metadata agent - After=nss-lookup.target - After=network-online.target - Wants=network-online.target - [Service] - Type=oneshot - Restart=on-failure - RemainAfterExit=yes - Environment=OUTPUT=/run/metadata/coreos - ExecStart=/usr/bin/mkdir --parent /run/metadata - ExecStart=/usr/bin/bash -cv 'echo "COREOS_CUSTOM_HOSTNAME=$(/usr/share/oem/bin/vmtoolsd --cmd "info-get guestinfo.metadata" | base64 -d | grep local-hostname | awk {\'print $2\'} | tr -d \'"\')" > $${OUTPUT}' - - name: set-hostname.service - enabled: true - contents: | - [Unit] - Description=Set the hostname for this machine - Requires=coreos-metadata.service - After=coreos-metadata.service - [Service] - Type=oneshot - EnvironmentFile=/run/metadata/coreos - ExecStart=/opt/set-hostname - [Install] - WantedBy=multi-user.target - - name: kubeadm.service - enabled: true - dropins: - - name: 10-flatcar.conf - contents: | - [Unit] - # kubeadm must run after coreos-metadata populated /run/metadata directory. - Requires=coreos-metadata.service - After=coreos-metadata.service - # kubeadm must run after containerd - see https://github.com/kubernetes-sigs/image-builder/issues/939. - After=containerd.service - [Service] - # Make metadata environment variables available for pre-kubeadm commands. - EnvironmentFile=/run/metadata/* - joinConfiguration: - nodeRegistration: - criSocket: /var/run/containerd/containerd.sock - kubeletExtraArgs: - cloud-provider: external - name: $${COREOS_CUSTOM_HOSTNAME} - preKubeadmCommands: - - envsubst < /etc/kubeadm.yml > /etc/kubeadm.yml.tmp - - mv /etc/kubeadm.yml.tmp /etc/kubeadm.yml - users: - - name: core - sshAuthorizedKeys: - - '${VSPHERE_SSH_AUTHORIZED_KEY}' - sudo: ALL=(ALL) NOPASSWD:ALL ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: VSphereMachineTemplate -metadata: - name: '${CLUSTER_NAME}' - namespace: '${NAMESPACE}' -spec: - template: - spec: - template: '${FLATCAR_VSPHERE_TEMPLATE}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: VSphereMachineTemplate -metadata: - name: ${CLUSTER_NAME}-worker - namespace: '${NAMESPACE}' -spec: - template: - spec: - template: '${FLATCAR_VSPHERE_TEMPLATE}' diff --git a/test/e2e/data/infrastructure-vsphere/main/ignition/kustomization.yaml b/test/e2e/data/infrastructure-vsphere/main/ignition/kustomization.yaml index 4d3ac9a02c..6412c24716 100644 --- a/test/e2e/data/infrastructure-vsphere/main/ignition/kustomization.yaml +++ b/test/e2e/data/infrastructure-vsphere/main/ignition/kustomization.yaml @@ -1,6 +1,16 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../base + - ../base/cluster-template-ignition.yaml + - ../commons/cluster-resource-set.yaml patchesStrategicMerge: - - ignition.yaml + - ../commons/cluster-resource-set-label.yaml + - ../commons/cluster-network-CIDR.yaml + - ../commons/cluster-resource-set-csi-insecure.yaml + - flatcar.yaml +patches: + # We are dropping storage policy so we also have test coverage + # for normal provisioning via data stores. + - target: + kind: VSphereMachineTemplate + path: ../commons/remove-storage-policy.yaml