From 801186ae5f5138809b61e1d32914c664422883db Mon Sep 17 00:00:00 2001 From: Guillermo Gaston Date: Fri, 15 Sep 2023 14:24:57 -0500 Subject: [PATCH] Separate etcd upgrade changes from kcp changes (#6496) In order to have more control over the upgrade process and make sure than in case of error the cleanup is easier, we upgrade etcd separately from the CP nodes for external etcd topologies. The controller will apply etcd changes in isolation and wait for etcd to be ready before moving on with the CP nodes. --- pkg/controller/clusters/controlplane.go | 176 ++++++++++++++---- pkg/controller/clusters/controlplane_test.go | 127 ++++++++++--- .../cloudstack/reconciler/reconciler.go | 2 +- pkg/providers/docker/reconciler/reconciler.go | 2 +- .../nutanix/reconciler/reconciler.go | 2 +- pkg/providers/snow/reconciler/reconciler.go | 2 +- .../tinkerbell/reconciler/reconciler.go | 2 +- .../vsphere/reconciler/reconciler.go | 2 +- 8 files changed, 247 insertions(+), 68 deletions(-) diff --git a/pkg/controller/clusters/controlplane.go b/pkg/controller/clusters/controlplane.go index 0aac274a49c1..1b94ca2e73e6 100644 --- a/pkg/controller/clusters/controlplane.go +++ b/pkg/controller/clusters/controlplane.go @@ -3,13 +3,18 @@ package clusters import ( "context" "reflect" + "time" etcdv1 "github.com/aws/etcdadm-controller/api/v1beta1" + "github.com/go-logr/logr" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/eks-anywhere/pkg/controller" @@ -44,29 +49,48 @@ type ControlPlane struct { // AllObjects returns all the control plane objects. func (cp *ControlPlane) AllObjects() []client.Object { - objs := make([]client.Object, 0, 6+len(cp.Other)) + objs := cp.nonEtcdObjects() + if cp.EtcdCluster != nil { + objs = append(objs, cp.etcdObjects()...) + } + + return objs +} + +func (cp *ControlPlane) etcdObjects() []client.Object { + return []client.Object{cp.EtcdMachineTemplate, cp.EtcdCluster} +} + +func (cp *ControlPlane) nonEtcdObjects() []client.Object { + objs := make([]client.Object, 0, 4+len(cp.Other)) objs = append(objs, cp.Cluster, cp.ProviderCluster, cp.KubeadmControlPlane) if !reflect.ValueOf(cp.ControlPlaneMachineTemplate).IsNil() { objs = append(objs, cp.ControlPlaneMachineTemplate) } - if cp.EtcdCluster != nil { - objs = append(objs, cp.EtcdCluster, cp.EtcdMachineTemplate) - } objs = append(objs, cp.Other...) return objs } +// skipCAPIAutoPauseKCPForExternalEtcdAnnotation instructs the CAPI cluster controller to not pause or unpause +// the KCP to wait for etcd endpoints to be ready. When this annotation is present, is left to the user (us) +// to orchestrate this operation if double kcp rollouts are undesirable. +const skipCAPIAutoPauseKCPForExternalEtcdAnnotation = "cluster.x-k8s.io/skip-pause-cp-managed-etcd" + // ReconcileControlPlane orchestrates the ControlPlane reconciliation logic. -func ReconcileControlPlane(ctx context.Context, c client.Client, cp *ControlPlane) (controller.Result, error) { +func ReconcileControlPlane(ctx context.Context, log logr.Logger, c client.Client, cp *ControlPlane) (controller.Result, error) { if cp.EtcdCluster == nil { // For stacked etcd, we don't need orchestration, apply directly return controller.Result{}, applyAllControlPlaneObjects(ctx, c, cp) } + // always add skip pause annotation since we want to have full control over the kcp-etcd orchestration + clientutil.AddAnnotation(cp.KubeadmControlPlane, skipCAPIAutoPauseKCPForExternalEtcdAnnotation, "true") + cluster := &clusterv1.Cluster{} err := c.Get(ctx, client.ObjectKeyFromObject(cp.Cluster), cluster) if apierrors.IsNotFound(err) { + log.Info("Creating cluster with external etcd") // If the CAPI cluster doesn't exist, this is a new cluster, create all objects return controller.Result{}, applyAllControlPlaneObjects(ctx, c, cp) } @@ -74,61 +98,131 @@ func ReconcileControlPlane(ctx context.Context, c client.Client, cp *ControlPlan return controller.Result{}, errors.Wrap(err, "reading CAPI cluster") } - externalEtcdNamespace := cluster.Spec.ManagedExternalEtcdRef.Namespace - // This can happen when a user has a workload cluster that is older than the following PR, causing cluster - // reconcilation to fail. By inferring namespace from clusterv1.Cluster, we will be able to retrieve the object correctly. - // PR: https://github.com/aws/eks-anywhere/pull/4025 - // TODO: See if it is possible to propagate the namespace field in the clusterv1.Cluster object in cluster-api like the other refs. - if externalEtcdNamespace == "" { - externalEtcdNamespace = cluster.Namespace + etcdadmCluster, err := getEtcdadmCluster(ctx, c, cluster) + if err != nil { + return controller.Result{}, errors.Wrap(err, "reading CAPI cluster") } - etcdadmCluster := &etcdv1.EtcdadmCluster{} - key := client.ObjectKey{ - Name: cluster.Spec.ManagedExternalEtcdRef.Name, - Namespace: externalEtcdNamespace, - } - if err = c.Get(ctx, key, etcdadmCluster); err != nil { - return controller.Result{}, errors.Wrap(err, "reading etcdadm cluster") + kcp := &controlplanev1.KubeadmControlPlane{} + if err = c.Get(ctx, objKeyForRef(cluster.Spec.ControlPlaneRef), kcp); err != nil { + return controller.Result{}, errors.Wrap(err, "reading kubeadm control plane") } + // If there are changes for etcd, we only apply those changes for now and we wait. if !equality.Semantic.DeepDerivative(cp.EtcdCluster.Spec, etcdadmCluster.Spec) { - // If the etcdadm cluster has changes, this will require a rolling upgrade - // Mark the etcdadm cluster as upgrading and pause the kcp reconciliation - // The CAPI cluster and etcdadm cluster controller will take care of removing - // these annotation at the right time to orchestrate the kcp upgrade - clientutil.AddAnnotation(cp.EtcdCluster, etcdv1.UpgradeInProgressAnnotation, "true") - clientutil.AddAnnotation(cp.KubeadmControlPlane, clusterv1.PausedAnnotation, "true") + return reconcileEtcdChanges(ctx, log, c, cp, kcp, etcdadmCluster) + } + + // If etcd is not ready yet, we requeue and wait before making any other change to the control plane. + if !etcdadmClusterReady(etcdadmCluster) { + // We need to inject a logger in this method or extract from context + log.Info("Etcd is not ready, requeuing") + return controller.ResultWithRequeue(30 * time.Second), nil } + return reconcileControlPlaneNodeChanges(ctx, log, c, cp, kcp) +} + +func applyAllControlPlaneObjects(ctx context.Context, c client.Client, cp *ControlPlane) error { + if err := serverside.ReconcileObjects(ctx, c, cp.AllObjects()); err != nil { + return errors.Wrap(err, "applying control plane objects") + } + + return nil +} + +func reconcileEtcdChanges(ctx context.Context, log logr.Logger, c client.Client, desiredCP *ControlPlane, currentKCP *controlplanev1.KubeadmControlPlane, currentEtcdadmCluster *etcdv1.EtcdadmCluster) (controller.Result, error) { + // Before making any changes to etcd, pause the KCP so it doesn't rollout new nodes as the + // etcd endpoints change. + if !annotations.HasPaused(currentKCP) { + log.Info("Pausing KCP before making any etcd changes", "kcp", klog.KObj(currentKCP)) + clientutil.AddAnnotation(currentKCP, clusterv1.PausedAnnotation, "true") + if err := c.Update(ctx, currentKCP); err != nil { + return controller.Result{}, err + } + } + + // If the etcdadm cluster has changes, this will require a rolling upgrade + // Mark the etcdadm cluster as upgrading + // The etcdadm cluster controller will take care of removing + // this annotation at the right time to orchestrate the kcp upgrade. + clientutil.AddAnnotation(desiredCP.EtcdCluster, etcdv1.UpgradeInProgressAnnotation, "true") + + log.Info("Reconciling external etcd changes", "etcdadmCluster", klog.KObj(currentEtcdadmCluster)) + if err := serverside.ReconcileObjects(ctx, c, desiredCP.etcdObjects()); err != nil { + return controller.Result{}, errors.Wrap(err, "applying etcd objects") + } + + // After applying etcd changes, just requeue to wait until etcd finishes updating and is ready + // We use a short wait here just in case etcdadm controller decides that not new machines are + // needed. + log.Info("Requeuing to wait until etcd is ready") + return controller.ResultWithRequeue(10 * time.Second), nil +} + +func etcdadmClusterReady(etcdadmCluster *etcdv1.EtcdadmCluster) bool { + // It's important to use status.Ready and not the Ready condition, since the Ready condition + // only becomes true after the old etcd members have been deleted, which only happens after the + // kcp finishes its own upgrade. + return etcdadmCluster.Generation == etcdadmCluster.Status.ObservedGeneration && etcdadmCluster.Status.Ready +} + +func reconcileControlPlaneNodeChanges(ctx context.Context, log logr.Logger, c client.Client, desiredCP *ControlPlane, currentKCP *controlplanev1.KubeadmControlPlane) (controller.Result, error) { // When the controller reconciles the control plane for a cluster with an external etcd configuration // the KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints field is // defaulted to an empty slice. However, at some point that field in KubeadmControlPlane object is filled - // and updated by another component - + // and updated by the kcp controller. + // // We do not want to update the field with an empty slice again, so here we check if the endpoints for the // external etcd have already been populated on the KubeadmControlPlane object and override ours before applying it. - kcp := &controlplanev1.KubeadmControlPlane{} - kcpKey := client.ObjectKey{ - Name: cluster.Spec.ControlPlaneRef.Name, - Namespace: cluster.Spec.ControlPlaneRef.Namespace, + externalEndpoints := currentKCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints + if len(externalEndpoints) != 0 { + desiredCP.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = externalEndpoints } - if err = c.Get(ctx, kcpKey, kcp); err != nil { - return controller.Result{}, errors.Wrap(err, "reading kubeadmcontrolplane object") + + if err := serverside.ReconcileObjects(ctx, c, desiredCP.nonEtcdObjects()); err != nil { + return controller.Result{}, errors.Wrap(err, "applying non etcd control plane objects") } - externalEndpoints := kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints - if len(externalEndpoints) != 0 { - cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = externalEndpoints + // If the KCP is paused, we read the last version (in case we just updated it) and unpause it + // so the cp nodes are reconciled. + if annotations.HasPaused(currentKCP) { + kcp := &controlplanev1.KubeadmControlPlane{} + if err := c.Get(ctx, client.ObjectKeyFromObject(currentKCP), kcp); err != nil { + return controller.Result{}, errors.Wrap(err, "reading updates kubeadm control plane to unpause") + } + + delete(kcp.Annotations, clusterv1.PausedAnnotation) + log.Info("Unpausing KCP after update to start reconciliation", klog.KObj(currentKCP)) + if err := c.Update(ctx, kcp); err != nil { + return controller.Result{}, err + } } - return controller.Result{}, applyAllControlPlaneObjects(ctx, c, cp) + return controller.Result{}, nil } -func applyAllControlPlaneObjects(ctx context.Context, c client.Client, cp *ControlPlane) error { - if err := serverside.ReconcileObjects(ctx, c, cp.AllObjects()); err != nil { - return errors.Wrap(err, "applying control plane objects") +func getEtcdadmCluster(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) (*etcdv1.EtcdadmCluster, error) { + key := objKeyForRef(cluster.Spec.ManagedExternalEtcdRef) + // This can happen when a user has a workload cluster that is older than the following PR, causing cluster + // reconcilation to fail. By inferring namespace from clusterv1.Cluster, we will be able to retrieve the object correctly. + // PR: https://github.com/aws/eks-anywhere/pull/4025 + // TODO: See if it is possible to propagate the namespace field in the clusterv1.Cluster object in cluster-api like the other refs. + if key.Namespace == "" { + key.Namespace = cluster.Namespace } - return nil + etcdadmCluster := &etcdv1.EtcdadmCluster{} + if err := c.Get(ctx, key, etcdadmCluster); err != nil { + return nil, errors.Wrap(err, "reading etcdadm cluster") + } + + return etcdadmCluster, nil +} + +func objKeyForRef(ref *corev1.ObjectReference) client.ObjectKey { + return client.ObjectKey{ + Name: ref.Name, + Namespace: ref.Namespace, + } } diff --git a/pkg/controller/clusters/controlplane_test.go b/pkg/controller/clusters/controlplane_test.go index ddb8ef43bd89..3216ce052890 100644 --- a/pkg/controller/clusters/controlplane_test.go +++ b/pkg/controller/clusters/controlplane_test.go @@ -3,6 +3,7 @@ package clusters_test import ( "context" "testing" + "time" etcdv1 "github.com/aws/etcdadm-controller/api/v1beta1" . "github.com/onsi/gomega" @@ -12,10 +13,14 @@ import ( "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" dockerv1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" + "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/aws/eks-anywhere/internal/test" "github.com/aws/eks-anywhere/internal/test/envtest" "github.com/aws/eks-anywhere/pkg/controller" + "github.com/aws/eks-anywhere/pkg/controller/clientutil" "github.com/aws/eks-anywhere/pkg/controller/clusters" "github.com/aws/eks-anywhere/pkg/utils/ptr" ) @@ -26,9 +31,10 @@ func TestReconcileControlPlaneStackedEtcd(t *testing.T) { api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneStackedEtcd(ns) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To(Equal(controller.Result{})) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) @@ -41,15 +47,28 @@ func TestReconcileControlPlaneExternalEtcdNewCluster(t *testing.T) { api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To(Equal(controller.Result{})) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) + + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) + api.ShouldEventuallyMatch( + ctx, + kcp, + func(g Gomega) { + g.Expect(kcp.Annotations).To( + HaveKey("cluster.x-k8s.io/skip-pause-cp-managed-etcd"), + "kcp should have skip pause annotation after being created", + ) + }, + ) } func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { @@ -58,12 +77,21 @@ func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) + + var oldCPReplicas int32 = 3 + var newCPReplicas int32 = 4 + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(oldCPReplicas) + envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) cp.EtcdCluster.Spec.Replicas = ptr.Int32(5) + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(newCPReplicas) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To( + Equal(controller.Result{Result: &reconcile.Result{RequeueAfter: 10 * time.Second}}), + ) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) @@ -71,7 +99,7 @@ func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) - etcdadmCluster := &etcdv1.EtcdadmCluster{ObjectMeta: cp.EtcdCluster.ObjectMeta} + etcdadmCluster := envtest.CloneNameNamespace(cp.EtcdCluster) api.ShouldEventuallyMatch( ctx, etcdadmCluster, @@ -83,80 +111,137 @@ func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { ) }, ) - kcp := &etcdv1.EtcdadmCluster{ObjectMeta: cp.KubeadmControlPlane.ObjectMeta} + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) api.ShouldEventuallyMatch( ctx, - etcdadmCluster, + kcp, func(g Gomega) { g.Expect(kcp.Annotations).To( HaveKeyWithValue(clusterv1.PausedAnnotation, "true"), "kcp paused annotation should have been added", ) + g.Expect(kcp.Spec.Replicas).To( + HaveValue(Equal(oldCPReplicas)), + "kcp replicas should not have changed", + ) }, ) } -func TestReconcileControlPlaneExternalEtcdUpgradeWithNoDiff(t *testing.T) { +func TestReconcileControlPlaneExternalEtcdNotReady(t *testing.T) { g := NewWithT(t) c := env.Client() api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) + var oldCPReplicas int32 = 3 + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(oldCPReplicas) envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(4) + + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To( + Equal(controller.Result{Result: &reconcile.Result{RequeueAfter: 30 * time.Second}}), + ) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) + + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) + api.ShouldEventuallyMatch( + ctx, + kcp, + func(g Gomega) { + g.Expect(kcp.Spec.Replicas).To( + HaveValue(Equal(oldCPReplicas)), + "kcp replicas should not have changed", + ) + }, + ) } -func TestReconcileControlPlaneExternalEtcdUpgradeWithNoNamespace(t *testing.T) { +func TestReconcileControlPlaneExternalEtcdReadyControlPlaneUpgrade(t *testing.T) { g := NewWithT(t) c := env.Client() api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) - cp.Cluster.Spec.ManagedExternalEtcdRef.Namespace = "" + cp.EtcdCluster.Status.Ready = true + cp.EtcdCluster.Status.ObservedGeneration = 1 + + var oldCPReplicas int32 = 3 + var newCPReplicas int32 = 4 + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(oldCPReplicas) + clientutil.AddAnnotation(cp.KubeadmControlPlane, clusterv1.PausedAnnotation, "true") + // an existing kcp should already have etcd endpoints + cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{"https://1.1.1.1:2379"} envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(newCPReplicas) + // by default providers code will generate kcp with empty endpoints, so we imitate that here + cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{} + + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To( + Equal(controller.Result{}), + ) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) + + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) + api.ShouldEventuallyMatch( + ctx, + kcp, + func(g Gomega) { + g.Expect(kcp.Annotations).To( + HaveKey("cluster.x-k8s.io/skip-pause-cp-managed-etcd"), + "kcp should have skip pause annotation", + ) + g.Expect(annotations.HasPaused(kcp)).To( + BeFalse(), "kcp should not be paused", + ) + g.Expect(kcp.Spec.Replicas).To( + HaveValue(Equal(newCPReplicas)), + "kcp replicas should have been updated", + ) + g.Expect(kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints).To( + HaveExactElements("https://1.1.1.1:2379"), + "Etcd endpoints should remain the same and not be emptied", + ) + }, + ) } -func TestReconcileControlPlaneExternalEtcdWithExistingEndpoints(t *testing.T) { +func TestReconcileControlPlaneExternalEtcdUpgradeWithNoNamespace(t *testing.T) { g := NewWithT(t) c := env.Client() api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) - cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{"https://1.1.1.1:2379"} + cp.Cluster.Spec.ManagedExternalEtcdRef.Namespace = "" + cp.EtcdCluster.Status.Ready = true + cp.EtcdCluster.Status.ObservedGeneration = 1 envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) - cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{} - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To(Equal(controller.Result{})) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) - - kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) - api.ShouldEventuallyMatch(ctx, kcp, func(g Gomega) { - endpoints := kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints - g.Expect(endpoints).To(ContainElement("https://1.1.1.1:2379")) - }) } func controlPlaneStackedEtcd(namespace string) *clusters.ControlPlane { diff --git a/pkg/providers/cloudstack/reconciler/reconciler.go b/pkg/providers/cloudstack/reconciler/reconciler.go index 7efbc954dc5a..f0c7ed564733 100644 --- a/pkg/providers/cloudstack/reconciler/reconciler.go +++ b/pkg/providers/cloudstack/reconciler/reconciler.go @@ -125,7 +125,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, &clusters.ControlPlane{ + return clusters.ReconcileControlPlane(ctx, log, r.client, &clusters.ControlPlane{ Cluster: cp.Cluster, ProviderCluster: cp.ProviderCluster, KubeadmControlPlane: cp.KubeadmControlPlane, diff --git a/pkg/providers/docker/reconciler/reconciler.go b/pkg/providers/docker/reconciler/reconciler.go index 810a16c6098f..75724e49f8ed 100644 --- a/pkg/providers/docker/reconciler/reconciler.go +++ b/pkg/providers/docker/reconciler/reconciler.go @@ -112,7 +112,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, if err != nil { return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, &clusters.ControlPlane{ + return clusters.ReconcileControlPlane(ctx, log, r.client, &clusters.ControlPlane{ Cluster: cp.Cluster, ProviderCluster: cp.ProviderCluster, KubeadmControlPlane: cp.KubeadmControlPlane, diff --git a/pkg/providers/nutanix/reconciler/reconciler.go b/pkg/providers/nutanix/reconciler/reconciler.go index 63b692773e53..830dbf73ff9a 100644 --- a/pkg/providers/nutanix/reconciler/reconciler.go +++ b/pkg/providers/nutanix/reconciler/reconciler.go @@ -194,7 +194,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, toClientControlPlane(cp)) + return clusters.ReconcileControlPlane(ctx, log, r.client, toClientControlPlane(cp)) } func toClientControlPlane(cp *nutanix.ControlPlane) *clusters.ControlPlane { diff --git a/pkg/providers/snow/reconciler/reconciler.go b/pkg/providers/snow/reconciler/reconciler.go index 0b938d05a630..495a5986ab36 100644 --- a/pkg/providers/snow/reconciler/reconciler.go +++ b/pkg/providers/snow/reconciler/reconciler.go @@ -111,7 +111,7 @@ func (s *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, s.client, toClientControlPlane(cp)) + return clusters.ReconcileControlPlane(ctx, log, s.client, toClientControlPlane(cp)) } func (r *Reconciler) CheckControlPlaneReady(ctx context.Context, log logr.Logger, clusterSpec *cluster.Spec) (controller.Result, error) { diff --git a/pkg/providers/tinkerbell/reconciler/reconciler.go b/pkg/providers/tinkerbell/reconciler/reconciler.go index d1264b69690f..7cd48064004d 100644 --- a/pkg/providers/tinkerbell/reconciler/reconciler.go +++ b/pkg/providers/tinkerbell/reconciler/reconciler.go @@ -235,7 +235,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, log = log.WithValues("phase", "reconcileControlPlane") log.Info("Applying control plane CAPI objects") - return clusters.ReconcileControlPlane(ctx, r.client, toClientControlPlane(tinkerbellScope.ControlPlane)) + return clusters.ReconcileControlPlane(ctx, log, r.client, toClientControlPlane(tinkerbellScope.ControlPlane)) } // CheckControlPlaneReady checks whether the control plane for an eks-a cluster is ready or not. diff --git a/pkg/providers/vsphere/reconciler/reconciler.go b/pkg/providers/vsphere/reconciler/reconciler.go index 1fe1fc624251..26c5676f50b8 100644 --- a/pkg/providers/vsphere/reconciler/reconciler.go +++ b/pkg/providers/vsphere/reconciler/reconciler.go @@ -192,7 +192,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, toClientControlPlane(cp)) + return clusters.ReconcileControlPlane(ctx, log, r.client, toClientControlPlane(cp)) } // CheckControlPlaneReady checks whether the control plane for an eks-a cluster is ready or not.