From e696bf340fd65383a6bdc3d77a596eadd0f683f1 Mon Sep 17 00:00:00 2001 From: Aravind Ramalingam Date: Mon, 4 Mar 2024 07:44:42 -0800 Subject: [PATCH] Adding patch for BR multiple reg mirrors --- .../kubernetes-sigs/cluster-api/CHECKSUMS | 20 +- ...or-multiple-registry-mirrors-in-bott.patch | 660 ++++++++++++++++++ ...troller-reconcilation-with-in-place.patch} | 0 ...-runtime-patch-for-restmapper-cache.patch} | 0 ...able-kubeadm-feature-flags-mutation.patch} | 0 5 files changed, 670 insertions(+), 10 deletions(-) create mode 100644 projects/kubernetes-sigs/cluster-api/patches/0039-Adding-support-for-multiple-registry-mirrors-in-bott.patch rename projects/kubernetes-sigs/cluster-api/patches/{0039-Fix-MD-controller-reconcilation-with-in-place.patch => 0040-Fix-MD-controller-reconcilation-with-in-place.patch} (100%) rename projects/kubernetes-sigs/cluster-api/patches/{0040-Use-controller-runtime-patch-for-restmapper-cache.patch => 0041-Use-controller-runtime-patch-for-restmapper-cache.patch} (100%) rename projects/kubernetes-sigs/cluster-api/patches/{0041-enable-kubeadm-feature-flags-mutation.patch => 0042-enable-kubeadm-feature-flags-mutation.patch} (100%) diff --git a/projects/kubernetes-sigs/cluster-api/CHECKSUMS b/projects/kubernetes-sigs/cluster-api/CHECKSUMS index 9f2858acf8..df00058e80 100644 --- a/projects/kubernetes-sigs/cluster-api/CHECKSUMS +++ b/projects/kubernetes-sigs/cluster-api/CHECKSUMS @@ -1,10 +1,10 @@ -02aa60008eefd978e073a54f175b9d643ce08b7f1337120ef23017d16349240e _output/bin/cluster-api/linux-amd64/cluster-api-provider-docker-manager -76d919cbb8db00b59b28d2b4a07b8e191bfeffaa7b9e3b1486d992ba51f19d20 _output/bin/cluster-api/linux-amd64/clusterctl -340b759340769c76aa34efb878d1b6fa779018a23f96234e79cf21c0a44143f6 _output/bin/cluster-api/linux-amd64/kubeadm-bootstrap-manager -4e7cf41fb10c1c2cafd42a1f7e13149a59b331aeceeff95a80407bbceb55c6b5 _output/bin/cluster-api/linux-amd64/kubeadm-control-plane-manager -3ab51d637374a08f9ebe2494dd75f3a0facbf12f71cd86a3a0f36e29a7df41c9 _output/bin/cluster-api/linux-amd64/manager -46faa0e313a910076b105c56fa2f2216b03e0ebe8c4cd62e2063b86f2a79fe26 _output/bin/cluster-api/linux-arm64/cluster-api-provider-docker-manager -9a077de1faab8cc7486437871bcc897de4c73ff091f6ccba3cc3d0203f443f69 _output/bin/cluster-api/linux-arm64/clusterctl -e20aba160867fd09182bb7499c99dc2ff4fe0eda1896e1aef1f90c039610922b _output/bin/cluster-api/linux-arm64/kubeadm-bootstrap-manager -4af6c9d1f8d08c269ff804f7f41192e4903c7a9a4ac941342d167722a6ea684b _output/bin/cluster-api/linux-arm64/kubeadm-control-plane-manager -7f9df625a074708a37a64fd9162398bdee9ae62ecbc7aae443dc980c0dbf05a8 _output/bin/cluster-api/linux-arm64/manager +ff54bcf90d1847aac0dbae8ba15c8d539a1b141534bca868c28a48548c6b2661 _output/bin/cluster-api/linux-amd64/cluster-api-provider-docker-manager +9d1202c52b79657ecad8ff306dbe2c2ef7bac5eed1d0dbd87eb74ac3c76d82aa _output/bin/cluster-api/linux-amd64/clusterctl +03447ee28c86576ddcccf2920ce4786762772cd85c44e572e321259d63b5c483 _output/bin/cluster-api/linux-amd64/kubeadm-bootstrap-manager +d758990ccd93f54200896738c5b60559055ad58774c9230061b8204fc210c045 _output/bin/cluster-api/linux-amd64/kubeadm-control-plane-manager +1c7d6747e3e085901d751e90ae0a45abbebc224a8c53c9ae143b0eaf5c2a8779 _output/bin/cluster-api/linux-amd64/manager +966c99b7709f1f095c7f55369e94ae9731c8f306a8f0473ba15fb66b04513f43 _output/bin/cluster-api/linux-arm64/cluster-api-provider-docker-manager +7d91014f9a4a3763d9a40d731bc3996c26845efb72b9b80b0e3c6febc9a6349e _output/bin/cluster-api/linux-arm64/clusterctl +eba5f4a17fe872343ba49b52a8b41b3dca284251bd036506ae2da83b9dd0e37e _output/bin/cluster-api/linux-arm64/kubeadm-bootstrap-manager +a067bf4926af5fd92a8465c297c0a2f1a0e2207b5d77a4ba55cc8835d87c8776 _output/bin/cluster-api/linux-arm64/kubeadm-control-plane-manager +cc461cadac46d544520f0eacb2d77d2f2fb5c325c464899d8abafe90e651d8f6 _output/bin/cluster-api/linux-arm64/manager diff --git a/projects/kubernetes-sigs/cluster-api/patches/0039-Adding-support-for-multiple-registry-mirrors-in-bott.patch b/projects/kubernetes-sigs/cluster-api/patches/0039-Adding-support-for-multiple-registry-mirrors-in-bott.patch new file mode 100644 index 0000000000..743c7ed5a8 --- /dev/null +++ b/projects/kubernetes-sigs/cluster-api/patches/0039-Adding-support-for-multiple-registry-mirrors-in-bott.patch @@ -0,0 +1,660 @@ +From bfe02ef55483c5cb0bdbb77877550a42695acbcd Mon Sep 17 00:00:00 2001 +From: Aravind Ramalingam +Date: Mon, 22 Jan 2024 12:47:26 -0800 +Subject: [PATCH] Adding support for multiple registry mirrors in bottlerocket + +--- + .../api/v1alpha4/zz_generated.conversion.go | 1 + + .../kubeadm/api/v1beta1/kubeadm_types.go | 15 ++++ + .../api/v1beta1/zz_generated.deepcopy.go | 34 ++++++++- + ...strap.cluster.x-k8s.io_kubeadmconfigs.yaml | 38 ++++++++++ + ...uster.x-k8s.io_kubeadmconfigtemplates.yaml | 42 +++++++++++ + .../internal/bottlerocket/bootstrap.go | 20 ++++-- + .../internal/bottlerocket/bootstrap_test.go | 72 +++++++++++++++++-- + .../internal/bottlerocket/bottlerocket.go | 44 +++++++++--- + .../controllers/kubeadmconfig_controller.go | 12 +++- + .../zz_generated.conversion.go | 1 + + .../zz_generated.conversion.go | 1 + + ...cluster.x-k8s.io_kubeadmcontrolplanes.yaml | 42 +++++++++++ + ...x-k8s.io_kubeadmcontrolplanetemplates.yaml | 44 ++++++++++++ + .../test/builder/zz_generated.deepcopy.go | 23 ++++++ + 14 files changed, 364 insertions(+), 25 deletions(-) + +diff --git a/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go b/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go +index d43c45841..6f0447507 100644 +--- a/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go ++++ b/bootstrap/kubeadm/api/v1alpha4/zz_generated.conversion.go +@@ -1720,6 +1720,7 @@ func Convert_v1alpha4_RegistryMirrorConfiguration_To_v1beta1_RegistryMirrorConfi + func autoConvert_v1beta1_RegistryMirrorConfiguration_To_v1alpha4_RegistryMirrorConfiguration(in *v1beta1.RegistryMirrorConfiguration, out *RegistryMirrorConfiguration, s conversion.Scope) error { + out.Endpoint = in.Endpoint + out.CACert = in.CACert ++ // INFO: in.Mirrors opted out of conversion generation + return nil + } + +diff --git a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go +index ed965709a..e3e4f3ef8 100644 +--- a/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go ++++ b/bootstrap/kubeadm/api/v1beta1/kubeadm_types.go +@@ -278,6 +278,21 @@ type RegistryMirrorConfiguration struct { + + // CACert defines the CA cert for the registry mirror + CACert string `json:"caCert,omitempty"` ++ ++ // Mirrors defines a list of image registry mirrors. ++ // +k8s:conversion-gen=false ++ // +optional ++ Mirrors []Mirror `json:"mirrors,omitempty"` ++} ++ ++// Mirror holds the settings for mirroring a registry. ++type Mirror struct { ++ // Registry defines the registry we are mirroring to the endpoint. ++ Registry string `json:"registry,omitempty"` ++ ++ // Endpoints defines the registry mirror endpoints to use for pulling images. ++ // Currently we support only one private registry. Hence endpoints would have only one entry. ++ Endpoints []string `json:"endpoints,omitempty"` + } + + // ControlPlaneComponent holds settings common to control plane component of the cluster. +diff --git a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go +index af85cfd21..7faf6f8c9 100644 +--- a/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go ++++ b/bootstrap/kubeadm/api/v1beta1/zz_generated.deepcopy.go +@@ -353,7 +354,7 @@ func (in *ClusterConfiguration) DeepCopyInto(out *ClusterConfiguration) { + out.BottlerocketAdmin = in.BottlerocketAdmin + out.BottlerocketControl = in.BottlerocketControl + in.Proxy.DeepCopyInto(&out.Proxy) +- out.RegistryMirror = in.RegistryMirror ++ in.RegistryMirror.DeepCopyInto(&out.RegistryMirror) + in.Etcd.DeepCopyInto(&out.Etcd) + out.Networking = in.Networking + in.APIServer.DeepCopyInto(&out.APIServer) +@@ -788,7 +789,7 @@ func (in *JoinConfiguration) DeepCopyInto(out *JoinConfiguration) { + out.BottlerocketAdmin = in.BottlerocketAdmin + out.BottlerocketControl = in.BottlerocketControl + in.Proxy.DeepCopyInto(&out.Proxy) +- out.RegistryMirror = in.RegistryMirror ++ in.RegistryMirror.DeepCopyInto(&out.RegistryMirror) + in.NodeRegistration.DeepCopyInto(&out.NodeRegistration) + in.Discovery.DeepCopyInto(&out.Discovery) + if in.ControlPlane != nil { +@@ -1157,6 +1158,26 @@ func (in *LocalEtcd) DeepCopy() *LocalEtcd { + return out + } + ++// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. ++func (in *Mirror) DeepCopyInto(out *Mirror) { ++ *out = *in ++ if in.Endpoints != nil { ++ in, out := &in.Endpoints, &out.Endpoints ++ *out = make([]string, len(*in)) ++ copy(*out, *in) ++ } ++} ++ ++// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Mirror. ++func (in *Mirror) DeepCopy() *Mirror { ++ if in == nil { ++ return nil ++ } ++ out := new(Mirror) ++ in.DeepCopyInto(out) ++ return out ++} ++ + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. + func (in MountPoints) DeepCopyInto(out *MountPoints) { + { +@@ -1345,6 +1366,13 @@ func (in *ProxyConfiguration) DeepCopy() *ProxyConfiguration { + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. + func (in *RegistryMirrorConfiguration) DeepCopyInto(out *RegistryMirrorConfiguration) { + *out = *in ++ if in.Mirrors != nil { ++ in, out := &in.Mirrors, &out.Mirrors ++ *out = make([]Mirror, len(*in)) ++ for i := range *in { ++ (*in)[i].DeepCopyInto(&(*out)[i]) ++ } ++ } + } + + // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RegistryMirrorConfiguration. +diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +index 9135db4f9..889745026 100644 +--- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml ++++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigs.yaml +@@ -1675,6 +1675,25 @@ spec: + description: Endpoint defines the registry mirror endpoint + to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry mirrors. ++ items: ++ description: Mirror holds the settings for mirroring a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry mirror endpoints ++ to use for pulling images. Currently we support only ++ one private registry. Hence endpoints would have only ++ one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry we are mirroring ++ to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + scheduler: + description: Scheduler contains extra settings for the scheduler +@@ -2528,6 +2547,25 @@ spec: + description: Endpoint defines the registry mirror endpoint + to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry mirrors. ++ items: ++ description: Mirror holds the settings for mirroring a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry mirror endpoints ++ to use for pulling images. Currently we support only ++ one private registry. Hence endpoints would have only ++ one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry we are mirroring ++ to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + skipPhases: + description: SkipPhases is a list of phases to skip during command +diff --git a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +index b097f1975..b9e0d00a1 100644 +--- a/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml ++++ b/bootstrap/kubeadm/config/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml +@@ -1737,6 +1737,27 @@ spec: + description: Endpoint defines the registry mirror + endpoint to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry ++ mirrors. ++ items: ++ description: Mirror holds the settings for mirroring ++ a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry ++ mirror endpoints to use for pulling images. ++ Currently we support only one private registry. ++ Hence endpoints would have only one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry we ++ are mirroring to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + scheduler: + description: Scheduler contains extra settings for the +@@ -2649,6 +2670,27 @@ spec: + description: Endpoint defines the registry mirror + endpoint to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry ++ mirrors. ++ items: ++ description: Mirror holds the settings for mirroring ++ a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry ++ mirror endpoints to use for pulling images. ++ Currently we support only one private registry. ++ Hence endpoints would have only one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry we ++ are mirroring to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + skipPhases: + description: SkipPhases is a list of phases to skip during +diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go +index df222b943..f378c3362 100644 +--- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go ++++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap.go +@@ -74,8 +74,11 @@ no-proxy = [{{stringsJoin .NoProxyEndpoints "," }}] + {{- end -}} + ` + registryMirrorTemplate = `{{ define "registryMirrorSettings" -}} +-[settings.container-registry.mirrors] +-"public.ecr.aws" = ["https://{{.RegistryMirrorEndpoint}}"] ++{{- range $orig, $mirror := .RegistryMirrorMap }} ++[[settings.container-registry.mirrors]] ++registry = "{{ $orig }}" ++endpoint = [{{stringsJoin $mirror "," }}] ++{{- end -}} + {{- end -}} + ` + registryMirrorCACertTemplate = `{{ define "registryMirrorCACertSettings" -}} +@@ -88,16 +91,21 @@ trusted=true + // to "public.ecr.aws" rather than the mirror's endpoint + // TODO: Once the bottlerocket fixes are in we need to remove the "public.ecr.aws" creds + registryMirrorCredentialsTemplate = `{{define "registryMirrorCredentialsSettings" -}} ++{{- range $orig, $mirror := .RegistryMirrorMap }} ++{{- if (eq $orig "public.ecr.aws")}} + [[settings.container-registry.credentials]] +-registry = "public.ecr.aws" +-username = "{{.RegistryMirrorUsername}}" +-password = "{{.RegistryMirrorPassword}}" ++registry = "{{ $orig }}" ++username = "{{$.RegistryMirrorUsername}}" ++password = "{{$.RegistryMirrorPassword}}" ++{{- end }} ++{{- end }} + [[settings.container-registry.credentials]] + registry = "{{.RegistryMirrorEndpoint}}" + username = "{{.RegistryMirrorUsername}}" + password = "{{.RegistryMirrorPassword}}" + {{- end -}} + ` ++ + nodeLabelsTemplate = `{{ define "nodeLabelSettings" -}} + [settings.kubernetes.node-labels] + {{.NodeLabels}} +@@ -153,7 +161,7 @@ trusted = true + {{- end -}} + + +-{{- if (ne .RegistryMirrorEndpoint "")}} ++{{- if .RegistryMirrorMap}} + {{template "registryMirrorSettings" .}} + {{- end -}} + +diff --git a/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go b/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go +index 2ee2d942b..ed58dc035 100644 +--- a/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go ++++ b/bootstrap/kubeadm/internal/bottlerocket/bootstrap_test.go +@@ -134,8 +134,10 @@ provider-id = "PROVIDERID" + + [settings.network] + hostname = "hostname" +-[settings.container-registry.mirrors] +-"public.ecr.aws" = ["https://REGISTRY_ENDPOINT"] ++ ++[[settings.container-registry.mirrors]] ++registry = "public.ecr.aws" ++endpoint = ["REGISTRY_ENDPOINT"] + [settings.pki.registry-mirror-ca] + data = "UkVHSVNUUllfQ0E=" + trusted=true` +@@ -162,11 +164,14 @@ provider-id = "PROVIDERID" + + [settings.network] + hostname = "hostname" +-[settings.container-registry.mirrors] +-"public.ecr.aws" = ["https://REGISTRY_ENDPOINT"] ++ ++[[settings.container-registry.mirrors]] ++registry = "public.ecr.aws" ++endpoint = ["REGISTRY_ENDPOINT"] + [settings.pki.registry-mirror-ca] + data = "UkVHSVNUUllfQ0E=" + trusted=true ++ + [[settings.container-registry.credentials]] + registry = "public.ecr.aws" + username = "admin" +@@ -340,6 +345,39 @@ trusted = true + [settings.pki.bundle2] + data = "MTIzNDU2" + trusted = true` ++ ++ registryMirrorMultipleMirrorsUserData = ` ++[settings.host-containers.admin] ++enabled = true ++superpowered = true ++source = "ADMIN_REPO:ADMIN_TAG" ++user-data = "CnsKCSJzc2giOiB7CgkJImF1dGhvcml6ZWQta2V5cyI6IFsic3NoLXJzYSBBQUEuLi4iXQoJfQp9" ++[settings.host-containers.kubeadm-bootstrap] ++enabled = true ++superpowered = true ++source = "BOOTSTRAP_REPO:BOOTSTRAP_TAG" ++user-data = "Qk9UVExFUk9DS0VUX0JPT1RTVFJBUF9VU0VSREFUQQ==" ++ ++[settings.kubernetes] ++cluster-domain = "cluster.local" ++standalone-mode = true ++authentication-mode = "tls" ++server-tls-bootstrap = false ++pod-infra-container-image = "PAUSE_REPO:PAUSE_TAG" ++provider-id = "PROVIDERID" ++ ++[settings.network] ++hostname = "hostname" ++ ++[[settings.container-registry.mirrors]] ++registry = "docker.io" ++endpoint = ["REGISTRY_ENDPOINT"] ++[[settings.container-registry.mirrors]] ++registry = "public.ecr.aws" ++endpoint = ["REGISTRY_ENDPOINT"] ++[settings.pki.registry-mirror-ca] ++data = "UkVHSVNUUllfQ0E=" ++trusted=true` + ) + + var ( +@@ -622,6 +660,32 @@ func TestGetBottlerocketNodeUserData(t *testing.T) { + }, + output: userDataWithCertBundle, + }, ++ { ++ name: "with multiple registries to mirror", ++ config: &BottlerocketConfig{ ++ BottlerocketAdmin: brAdmin, ++ BottlerocketBootstrap: brBootstrap, ++ Hostname: hostname, ++ Pause: pause, ++ KubeletExtraArgs: map[string]string{ ++ "provider-id": "PROVIDERID", ++ }, ++ RegistryMirrorConfiguration: bootstrapv1.RegistryMirrorConfiguration{ ++ CACert: "REGISTRY_CA", ++ Mirrors: []bootstrapv1.Mirror{ ++ { ++ Registry: "docker.io", ++ Endpoints: []string{"REGISTRY_ENDPOINT"}, ++ }, ++ { ++ Registry: "public.ecr.aws", ++ Endpoints: []string{"REGISTRY_ENDPOINT"}, ++ }, ++ }, ++ }, ++ }, ++ output: registryMirrorMultipleMirrorsUserData, ++ }, + } + for _, testcase := range testcases { + t.Run(testcase.name, func(t *testing.T) { +diff --git a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go +index 6e3aaaa95..3e38a7536 100644 +--- a/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go ++++ b/bootstrap/kubeadm/internal/bottlerocket/bottlerocket.go +@@ -4,6 +4,7 @@ import ( + "bytes" + "encoding/base64" + "fmt" ++ "regexp" + "strconv" + "strings" + "text/template" +@@ -63,6 +64,7 @@ type BottlerocketSettingsInput struct { + BootstrapContainers []bootstrapv1.BottlerocketBootstrapContainer + SysctlSettings string + CertBundles []bootstrapv1.CertBundle ++ RegistryMirrorMap map[string][]string + } + + // HostPath holds the path and type of a host path volume. +@@ -227,15 +229,14 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot + } + + bottlerocketInput := &BottlerocketSettingsInput{ +- PauseContainerSource: fmt.Sprintf("%s:%s", config.Pause.ImageRepository, config.Pause.ImageTag), +- HTTPSProxyEndpoint: config.ProxyConfiguration.HTTPSProxy, +- RegistryMirrorEndpoint: config.RegistryMirrorConfiguration.Endpoint, +- NodeLabels: parseNodeLabels(config.KubeletExtraArgs["node-labels"]), // empty string if it does not exist +- Taints: parseTaints(config.Taints), // empty string if it does not exist +- ProviderID: config.KubeletExtraArgs["provider-id"], +- Hostname: config.Hostname, +- HostContainers: hostContainers, +- BootstrapContainers: config.BottlerocketCustomBootstrapContainers, ++ PauseContainerSource: fmt.Sprintf("%s:%s", config.Pause.ImageRepository, config.Pause.ImageTag), ++ HTTPSProxyEndpoint: config.ProxyConfiguration.HTTPSProxy, ++ NodeLabels: parseNodeLabels(config.KubeletExtraArgs["node-labels"]), // empty string if it does not exist ++ Taints: parseTaints(config.Taints), // empty string if it does not exist ++ ProviderID: config.KubeletExtraArgs["provider-id"], ++ Hostname: config.Hostname, ++ HostContainers: hostContainers, ++ BootstrapContainers: config.BottlerocketCustomBootstrapContainers, + } + + if len(config.ProxyConfiguration.NoProxy) > 0 { +@@ -243,6 +244,31 @@ func getBottlerocketNodeUserData(bootstrapContainerUserData []byte, users []boot + bottlerocketInput.NoProxyEndpoints = append(bottlerocketInput.NoProxyEndpoints, strconv.Quote(noProxy)) + } + } ++ ++ // When RegistryMirrorConfiguration.Endpoint is specified, we default the mirror to public.ecr.aws. ++ // This was done for backward compatability, since public.ecr.aws was the only supported registry before. ++ // For existing customers this ensures that their nodes dont rollout, unless more mirrors are specified explicitly. ++ // If RegistryMirrorConfiguration.Endpoint is not specified, we iterate the RegistryMirrorConfiguration.Mirrors to setup the mirrors. ++ bottlerocketInput.RegistryMirrorMap = make(map[string][]string) ++ endpointRegex := regexp.MustCompile(`^(https?:\/\/)?[\w\.\:\-]+`) ++ if config.RegistryMirrorConfiguration.Endpoint != "" { ++ bottlerocketInput.RegistryMirrorMap["public.ecr.aws"] = []string{strconv.Quote(config.RegistryMirrorConfiguration.Endpoint)} ++ if endpoint := endpointRegex.FindStringSubmatch(config.RegistryMirrorConfiguration.Endpoint); endpoint != nil { ++ bottlerocketInput.RegistryMirrorEndpoint = endpoint[0] ++ } ++ } else if len(config.RegistryMirrorConfiguration.Mirrors) > 0 { ++ for _, mirror := range config.RegistryMirrorConfiguration.Mirrors { ++ for _, endpoint := range mirror.Endpoints { ++ bottlerocketInput.RegistryMirrorMap[mirror.Registry] = append(bottlerocketInput.RegistryMirrorMap[mirror.Registry], strconv.Quote(endpoint)) ++ } ++ } ++ ++ // Right now we support only one private registry. Hence defaulting to first entry. ++ if endpoint := endpointRegex.FindStringSubmatch(config.RegistryMirrorConfiguration.Mirrors[0].Endpoints[0]); endpoint != nil { ++ bottlerocketInput.RegistryMirrorEndpoint = endpoint[0] ++ } ++ } ++ + if config.RegistryMirrorConfiguration.CACert != "" { + bottlerocketInput.RegistryMirrorCACert = base64.StdEncoding.EncodeToString([]byte(config.RegistryMirrorConfiguration.CACert)) + } +diff --git a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +index f389a5056..51dc3cf19 100644 +--- a/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go ++++ b/bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go +@@ -489,7 +489,9 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex + if scope.Config.Spec.ClusterConfiguration.Proxy.HTTPSProxy != "" { + bottlerocketConfig.ProxyConfiguration = scope.Config.Spec.ClusterConfiguration.Proxy + } +- if scope.Config.Spec.ClusterConfiguration.RegistryMirror.Endpoint != "" { ++ ++ if scope.Config.Spec.ClusterConfiguration.RegistryMirror.Endpoint != "" || ++ len(scope.Config.Spec.ClusterConfiguration.RegistryMirror.Mirrors) != 0 { + bottlerocketConfig.RegistryMirrorConfiguration = scope.Config.Spec.ClusterConfiguration.RegistryMirror + registryUsername, registryPassword, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { +@@ -717,7 +719,9 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope) + if scope.Config.Spec.JoinConfiguration.Proxy.HTTPSProxy != "" { + bottlerocketConfig.ProxyConfiguration = scope.Config.Spec.JoinConfiguration.Proxy + } +- if scope.Config.Spec.JoinConfiguration.RegistryMirror.Endpoint != "" { ++ ++ if scope.Config.Spec.JoinConfiguration.RegistryMirror.Endpoint != "" || ++ len(scope.Config.Spec.JoinConfiguration.RegistryMirror.Mirrors) != 0 { + bottlerocketConfig.RegistryMirrorConfiguration = scope.Config.Spec.JoinConfiguration.RegistryMirror + registryUsername, registryPassword, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { +@@ -867,7 +871,9 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S + if scope.Config.Spec.JoinConfiguration.Proxy.HTTPSProxy != "" { + bottlerocketConfig.ProxyConfiguration = scope.Config.Spec.JoinConfiguration.Proxy + } +- if scope.Config.Spec.ClusterConfiguration.RegistryMirror.Endpoint != "" { ++ ++ if scope.Config.Spec.ClusterConfiguration.RegistryMirror.Endpoint != "" || ++ len(scope.Config.Spec.ClusterConfiguration.RegistryMirror.Mirrors) != 0 { + bottlerocketConfig.RegistryMirrorConfiguration = scope.Config.Spec.ClusterConfiguration.RegistryMirror + registryUsername, registryPassword, err := r.resolveRegistryCredentials(ctx, scope.Config) + if err != nil { +diff --git a/bootstrap/kubeadm/types/upstreamv1beta2/zz_generated.conversion.go b/bootstrap/kubeadm/types/upstreamv1beta2/zz_generated.conversion.go +index 3b12557c4..2f462fbe6 100644 +--- a/bootstrap/kubeadm/types/upstreamv1beta2/zz_generated.conversion.go ++++ b/bootstrap/kubeadm/types/upstreamv1beta2/zz_generated.conversion.go +@@ -1042,6 +1042,7 @@ func Convert_upstreamv1beta2_RegistryMirrorConfiguration_To_v1beta1_RegistryMirr + func autoConvert_v1beta1_RegistryMirrorConfiguration_To_upstreamv1beta2_RegistryMirrorConfiguration(in *v1beta1.RegistryMirrorConfiguration, out *RegistryMirrorConfiguration, s conversion.Scope) error { + out.Endpoint = in.Endpoint + out.CACert = in.CACert ++ // INFO: in.Mirrors opted out of conversion generation + return nil + } + +diff --git a/bootstrap/kubeadm/types/upstreamv1beta3/zz_generated.conversion.go b/bootstrap/kubeadm/types/upstreamv1beta3/zz_generated.conversion.go +index 6d812f516..dfff19e2d 100644 +--- a/bootstrap/kubeadm/types/upstreamv1beta3/zz_generated.conversion.go ++++ b/bootstrap/kubeadm/types/upstreamv1beta3/zz_generated.conversion.go +@@ -1055,6 +1055,7 @@ func Convert_upstreamv1beta3_RegistryMirrorConfiguration_To_v1beta1_RegistryMirr + func autoConvert_v1beta1_RegistryMirrorConfiguration_To_upstreamv1beta3_RegistryMirrorConfiguration(in *v1beta1.RegistryMirrorConfiguration, out *RegistryMirrorConfiguration, s conversion.Scope) error { + out.Endpoint = in.Endpoint + out.CACert = in.CACert ++ // INFO: in.Mirrors opted out of conversion generation + return nil + } + +diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +index 86f387bb9..6cfabed53 100644 +--- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml ++++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml +@@ -1971,6 +1971,27 @@ spec: + description: Endpoint defines the registry mirror endpoint + to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry ++ mirrors. ++ items: ++ description: Mirror holds the settings for mirroring ++ a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry mirror ++ endpoints to use for pulling images. Currently ++ we support only one private registry. Hence endpoints ++ would have only one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry we are ++ mirroring to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + scheduler: + description: Scheduler contains extra settings for the scheduler +@@ -2862,6 +2883,27 @@ spec: + description: Endpoint defines the registry mirror endpoint + to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry ++ mirrors. ++ items: ++ description: Mirror holds the settings for mirroring ++ a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry mirror ++ endpoints to use for pulling images. Currently ++ we support only one private registry. Hence endpoints ++ would have only one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry we are ++ mirroring to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + skipPhases: + description: SkipPhases is a list of phases to skip during +diff --git a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +index 9a2228488..2791830fd 100644 +--- a/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml ++++ b/controlplane/kubeadm/config/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanetemplates.yaml +@@ -1955,6 +1955,28 @@ spec: + description: Endpoint defines the registry mirror + endpoint to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry ++ mirrors. ++ items: ++ description: Mirror holds the settings for mirroring ++ a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry ++ mirror endpoints to use for pulling images. ++ Currently we support only one private ++ registry. Hence endpoints would have only ++ one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry ++ we are mirroring to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + scheduler: + description: Scheduler contains extra settings for +@@ -2907,6 +2929,28 @@ spec: + description: Endpoint defines the registry mirror + endpoint to use for pulling images + type: string ++ mirrors: ++ description: Mirrors defines a list of image registry ++ mirrors. ++ items: ++ description: Mirror holds the settings for mirroring ++ a registry. ++ properties: ++ endpoints: ++ description: Endpoints defines the registry ++ mirror endpoints to use for pulling images. ++ Currently we support only one private ++ registry. Hence endpoints would have only ++ one entry. ++ items: ++ type: string ++ type: array ++ registry: ++ description: Registry defines the registry ++ we are mirroring to the endpoint. ++ type: string ++ type: object ++ type: array + type: object + skipPhases: + description: SkipPhases is a list of phases to skip +-- +2.34.1 + diff --git a/projects/kubernetes-sigs/cluster-api/patches/0039-Fix-MD-controller-reconcilation-with-in-place.patch b/projects/kubernetes-sigs/cluster-api/patches/0040-Fix-MD-controller-reconcilation-with-in-place.patch similarity index 100% rename from projects/kubernetes-sigs/cluster-api/patches/0039-Fix-MD-controller-reconcilation-with-in-place.patch rename to projects/kubernetes-sigs/cluster-api/patches/0040-Fix-MD-controller-reconcilation-with-in-place.patch diff --git a/projects/kubernetes-sigs/cluster-api/patches/0040-Use-controller-runtime-patch-for-restmapper-cache.patch b/projects/kubernetes-sigs/cluster-api/patches/0041-Use-controller-runtime-patch-for-restmapper-cache.patch similarity index 100% rename from projects/kubernetes-sigs/cluster-api/patches/0040-Use-controller-runtime-patch-for-restmapper-cache.patch rename to projects/kubernetes-sigs/cluster-api/patches/0041-Use-controller-runtime-patch-for-restmapper-cache.patch diff --git a/projects/kubernetes-sigs/cluster-api/patches/0041-enable-kubeadm-feature-flags-mutation.patch b/projects/kubernetes-sigs/cluster-api/patches/0042-enable-kubeadm-feature-flags-mutation.patch similarity index 100% rename from projects/kubernetes-sigs/cluster-api/patches/0041-enable-kubeadm-feature-flags-mutation.patch rename to projects/kubernetes-sigs/cluster-api/patches/0042-enable-kubeadm-feature-flags-mutation.patch