diff --git a/pkg/controller/clusters/controlplane.go b/pkg/controller/clusters/controlplane.go index 0aac274a49c1..1b94ca2e73e6 100644 --- a/pkg/controller/clusters/controlplane.go +++ b/pkg/controller/clusters/controlplane.go @@ -3,13 +3,18 @@ package clusters import ( "context" "reflect" + "time" etcdv1 "github.com/aws/etcdadm-controller/api/v1beta1" + "github.com/go-logr/logr" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog/v2" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" + "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/eks-anywhere/pkg/controller" @@ -44,29 +49,48 @@ type ControlPlane struct { // AllObjects returns all the control plane objects. func (cp *ControlPlane) AllObjects() []client.Object { - objs := make([]client.Object, 0, 6+len(cp.Other)) + objs := cp.nonEtcdObjects() + if cp.EtcdCluster != nil { + objs = append(objs, cp.etcdObjects()...) + } + + return objs +} + +func (cp *ControlPlane) etcdObjects() []client.Object { + return []client.Object{cp.EtcdMachineTemplate, cp.EtcdCluster} +} + +func (cp *ControlPlane) nonEtcdObjects() []client.Object { + objs := make([]client.Object, 0, 4+len(cp.Other)) objs = append(objs, cp.Cluster, cp.ProviderCluster, cp.KubeadmControlPlane) if !reflect.ValueOf(cp.ControlPlaneMachineTemplate).IsNil() { objs = append(objs, cp.ControlPlaneMachineTemplate) } - if cp.EtcdCluster != nil { - objs = append(objs, cp.EtcdCluster, cp.EtcdMachineTemplate) - } objs = append(objs, cp.Other...) return objs } +// skipCAPIAutoPauseKCPForExternalEtcdAnnotation instructs the CAPI cluster controller to not pause or unpause +// the KCP to wait for etcd endpoints to be ready. When this annotation is present, is left to the user (us) +// to orchestrate this operation if double kcp rollouts are undesirable. +const skipCAPIAutoPauseKCPForExternalEtcdAnnotation = "cluster.x-k8s.io/skip-pause-cp-managed-etcd" + // ReconcileControlPlane orchestrates the ControlPlane reconciliation logic. -func ReconcileControlPlane(ctx context.Context, c client.Client, cp *ControlPlane) (controller.Result, error) { +func ReconcileControlPlane(ctx context.Context, log logr.Logger, c client.Client, cp *ControlPlane) (controller.Result, error) { if cp.EtcdCluster == nil { // For stacked etcd, we don't need orchestration, apply directly return controller.Result{}, applyAllControlPlaneObjects(ctx, c, cp) } + // always add skip pause annotation since we want to have full control over the kcp-etcd orchestration + clientutil.AddAnnotation(cp.KubeadmControlPlane, skipCAPIAutoPauseKCPForExternalEtcdAnnotation, "true") + cluster := &clusterv1.Cluster{} err := c.Get(ctx, client.ObjectKeyFromObject(cp.Cluster), cluster) if apierrors.IsNotFound(err) { + log.Info("Creating cluster with external etcd") // If the CAPI cluster doesn't exist, this is a new cluster, create all objects return controller.Result{}, applyAllControlPlaneObjects(ctx, c, cp) } @@ -74,61 +98,131 @@ func ReconcileControlPlane(ctx context.Context, c client.Client, cp *ControlPlan return controller.Result{}, errors.Wrap(err, "reading CAPI cluster") } - externalEtcdNamespace := cluster.Spec.ManagedExternalEtcdRef.Namespace - // This can happen when a user has a workload cluster that is older than the following PR, causing cluster - // reconcilation to fail. By inferring namespace from clusterv1.Cluster, we will be able to retrieve the object correctly. - // PR: https://github.com/aws/eks-anywhere/pull/4025 - // TODO: See if it is possible to propagate the namespace field in the clusterv1.Cluster object in cluster-api like the other refs. - if externalEtcdNamespace == "" { - externalEtcdNamespace = cluster.Namespace + etcdadmCluster, err := getEtcdadmCluster(ctx, c, cluster) + if err != nil { + return controller.Result{}, errors.Wrap(err, "reading CAPI cluster") } - etcdadmCluster := &etcdv1.EtcdadmCluster{} - key := client.ObjectKey{ - Name: cluster.Spec.ManagedExternalEtcdRef.Name, - Namespace: externalEtcdNamespace, - } - if err = c.Get(ctx, key, etcdadmCluster); err != nil { - return controller.Result{}, errors.Wrap(err, "reading etcdadm cluster") + kcp := &controlplanev1.KubeadmControlPlane{} + if err = c.Get(ctx, objKeyForRef(cluster.Spec.ControlPlaneRef), kcp); err != nil { + return controller.Result{}, errors.Wrap(err, "reading kubeadm control plane") } + // If there are changes for etcd, we only apply those changes for now and we wait. if !equality.Semantic.DeepDerivative(cp.EtcdCluster.Spec, etcdadmCluster.Spec) { - // If the etcdadm cluster has changes, this will require a rolling upgrade - // Mark the etcdadm cluster as upgrading and pause the kcp reconciliation - // The CAPI cluster and etcdadm cluster controller will take care of removing - // these annotation at the right time to orchestrate the kcp upgrade - clientutil.AddAnnotation(cp.EtcdCluster, etcdv1.UpgradeInProgressAnnotation, "true") - clientutil.AddAnnotation(cp.KubeadmControlPlane, clusterv1.PausedAnnotation, "true") + return reconcileEtcdChanges(ctx, log, c, cp, kcp, etcdadmCluster) + } + + // If etcd is not ready yet, we requeue and wait before making any other change to the control plane. + if !etcdadmClusterReady(etcdadmCluster) { + // We need to inject a logger in this method or extract from context + log.Info("Etcd is not ready, requeuing") + return controller.ResultWithRequeue(30 * time.Second), nil } + return reconcileControlPlaneNodeChanges(ctx, log, c, cp, kcp) +} + +func applyAllControlPlaneObjects(ctx context.Context, c client.Client, cp *ControlPlane) error { + if err := serverside.ReconcileObjects(ctx, c, cp.AllObjects()); err != nil { + return errors.Wrap(err, "applying control plane objects") + } + + return nil +} + +func reconcileEtcdChanges(ctx context.Context, log logr.Logger, c client.Client, desiredCP *ControlPlane, currentKCP *controlplanev1.KubeadmControlPlane, currentEtcdadmCluster *etcdv1.EtcdadmCluster) (controller.Result, error) { + // Before making any changes to etcd, pause the KCP so it doesn't rollout new nodes as the + // etcd endpoints change. + if !annotations.HasPaused(currentKCP) { + log.Info("Pausing KCP before making any etcd changes", "kcp", klog.KObj(currentKCP)) + clientutil.AddAnnotation(currentKCP, clusterv1.PausedAnnotation, "true") + if err := c.Update(ctx, currentKCP); err != nil { + return controller.Result{}, err + } + } + + // If the etcdadm cluster has changes, this will require a rolling upgrade + // Mark the etcdadm cluster as upgrading + // The etcdadm cluster controller will take care of removing + // this annotation at the right time to orchestrate the kcp upgrade. + clientutil.AddAnnotation(desiredCP.EtcdCluster, etcdv1.UpgradeInProgressAnnotation, "true") + + log.Info("Reconciling external etcd changes", "etcdadmCluster", klog.KObj(currentEtcdadmCluster)) + if err := serverside.ReconcileObjects(ctx, c, desiredCP.etcdObjects()); err != nil { + return controller.Result{}, errors.Wrap(err, "applying etcd objects") + } + + // After applying etcd changes, just requeue to wait until etcd finishes updating and is ready + // We use a short wait here just in case etcdadm controller decides that not new machines are + // needed. + log.Info("Requeuing to wait until etcd is ready") + return controller.ResultWithRequeue(10 * time.Second), nil +} + +func etcdadmClusterReady(etcdadmCluster *etcdv1.EtcdadmCluster) bool { + // It's important to use status.Ready and not the Ready condition, since the Ready condition + // only becomes true after the old etcd members have been deleted, which only happens after the + // kcp finishes its own upgrade. + return etcdadmCluster.Generation == etcdadmCluster.Status.ObservedGeneration && etcdadmCluster.Status.Ready +} + +func reconcileControlPlaneNodeChanges(ctx context.Context, log logr.Logger, c client.Client, desiredCP *ControlPlane, currentKCP *controlplanev1.KubeadmControlPlane) (controller.Result, error) { // When the controller reconciles the control plane for a cluster with an external etcd configuration // the KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints field is // defaulted to an empty slice. However, at some point that field in KubeadmControlPlane object is filled - // and updated by another component - + // and updated by the kcp controller. + // // We do not want to update the field with an empty slice again, so here we check if the endpoints for the // external etcd have already been populated on the KubeadmControlPlane object and override ours before applying it. - kcp := &controlplanev1.KubeadmControlPlane{} - kcpKey := client.ObjectKey{ - Name: cluster.Spec.ControlPlaneRef.Name, - Namespace: cluster.Spec.ControlPlaneRef.Namespace, + externalEndpoints := currentKCP.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints + if len(externalEndpoints) != 0 { + desiredCP.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = externalEndpoints } - if err = c.Get(ctx, kcpKey, kcp); err != nil { - return controller.Result{}, errors.Wrap(err, "reading kubeadmcontrolplane object") + + if err := serverside.ReconcileObjects(ctx, c, desiredCP.nonEtcdObjects()); err != nil { + return controller.Result{}, errors.Wrap(err, "applying non etcd control plane objects") } - externalEndpoints := kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints - if len(externalEndpoints) != 0 { - cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = externalEndpoints + // If the KCP is paused, we read the last version (in case we just updated it) and unpause it + // so the cp nodes are reconciled. + if annotations.HasPaused(currentKCP) { + kcp := &controlplanev1.KubeadmControlPlane{} + if err := c.Get(ctx, client.ObjectKeyFromObject(currentKCP), kcp); err != nil { + return controller.Result{}, errors.Wrap(err, "reading updates kubeadm control plane to unpause") + } + + delete(kcp.Annotations, clusterv1.PausedAnnotation) + log.Info("Unpausing KCP after update to start reconciliation", klog.KObj(currentKCP)) + if err := c.Update(ctx, kcp); err != nil { + return controller.Result{}, err + } } - return controller.Result{}, applyAllControlPlaneObjects(ctx, c, cp) + return controller.Result{}, nil } -func applyAllControlPlaneObjects(ctx context.Context, c client.Client, cp *ControlPlane) error { - if err := serverside.ReconcileObjects(ctx, c, cp.AllObjects()); err != nil { - return errors.Wrap(err, "applying control plane objects") +func getEtcdadmCluster(ctx context.Context, c client.Client, cluster *clusterv1.Cluster) (*etcdv1.EtcdadmCluster, error) { + key := objKeyForRef(cluster.Spec.ManagedExternalEtcdRef) + // This can happen when a user has a workload cluster that is older than the following PR, causing cluster + // reconcilation to fail. By inferring namespace from clusterv1.Cluster, we will be able to retrieve the object correctly. + // PR: https://github.com/aws/eks-anywhere/pull/4025 + // TODO: See if it is possible to propagate the namespace field in the clusterv1.Cluster object in cluster-api like the other refs. + if key.Namespace == "" { + key.Namespace = cluster.Namespace } - return nil + etcdadmCluster := &etcdv1.EtcdadmCluster{} + if err := c.Get(ctx, key, etcdadmCluster); err != nil { + return nil, errors.Wrap(err, "reading etcdadm cluster") + } + + return etcdadmCluster, nil +} + +func objKeyForRef(ref *corev1.ObjectReference) client.ObjectKey { + return client.ObjectKey{ + Name: ref.Name, + Namespace: ref.Namespace, + } } diff --git a/pkg/controller/clusters/controlplane_test.go b/pkg/controller/clusters/controlplane_test.go index ddb8ef43bd89..3216ce052890 100644 --- a/pkg/controller/clusters/controlplane_test.go +++ b/pkg/controller/clusters/controlplane_test.go @@ -3,6 +3,7 @@ package clusters_test import ( "context" "testing" + "time" etcdv1 "github.com/aws/etcdadm-controller/api/v1beta1" . "github.com/onsi/gomega" @@ -12,10 +13,14 @@ import ( "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" dockerv1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1" + "sigs.k8s.io/cluster-api/util/annotations" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/aws/eks-anywhere/internal/test" "github.com/aws/eks-anywhere/internal/test/envtest" "github.com/aws/eks-anywhere/pkg/controller" + "github.com/aws/eks-anywhere/pkg/controller/clientutil" "github.com/aws/eks-anywhere/pkg/controller/clusters" "github.com/aws/eks-anywhere/pkg/utils/ptr" ) @@ -26,9 +31,10 @@ func TestReconcileControlPlaneStackedEtcd(t *testing.T) { api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneStackedEtcd(ns) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To(Equal(controller.Result{})) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) @@ -41,15 +47,28 @@ func TestReconcileControlPlaneExternalEtcdNewCluster(t *testing.T) { api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To(Equal(controller.Result{})) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) + + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) + api.ShouldEventuallyMatch( + ctx, + kcp, + func(g Gomega) { + g.Expect(kcp.Annotations).To( + HaveKey("cluster.x-k8s.io/skip-pause-cp-managed-etcd"), + "kcp should have skip pause annotation after being created", + ) + }, + ) } func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { @@ -58,12 +77,21 @@ func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) + + var oldCPReplicas int32 = 3 + var newCPReplicas int32 = 4 + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(oldCPReplicas) + envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) cp.EtcdCluster.Spec.Replicas = ptr.Int32(5) + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(newCPReplicas) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To( + Equal(controller.Result{Result: &reconcile.Result{RequeueAfter: 10 * time.Second}}), + ) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) @@ -71,7 +99,7 @@ func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) - etcdadmCluster := &etcdv1.EtcdadmCluster{ObjectMeta: cp.EtcdCluster.ObjectMeta} + etcdadmCluster := envtest.CloneNameNamespace(cp.EtcdCluster) api.ShouldEventuallyMatch( ctx, etcdadmCluster, @@ -83,80 +111,137 @@ func TestReconcileControlPlaneExternalEtcdUpgradeWithDiff(t *testing.T) { ) }, ) - kcp := &etcdv1.EtcdadmCluster{ObjectMeta: cp.KubeadmControlPlane.ObjectMeta} + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) api.ShouldEventuallyMatch( ctx, - etcdadmCluster, + kcp, func(g Gomega) { g.Expect(kcp.Annotations).To( HaveKeyWithValue(clusterv1.PausedAnnotation, "true"), "kcp paused annotation should have been added", ) + g.Expect(kcp.Spec.Replicas).To( + HaveValue(Equal(oldCPReplicas)), + "kcp replicas should not have changed", + ) }, ) } -func TestReconcileControlPlaneExternalEtcdUpgradeWithNoDiff(t *testing.T) { +func TestReconcileControlPlaneExternalEtcdNotReady(t *testing.T) { g := NewWithT(t) c := env.Client() api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) + var oldCPReplicas int32 = 3 + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(oldCPReplicas) envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(4) + + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To( + Equal(controller.Result{Result: &reconcile.Result{RequeueAfter: 30 * time.Second}}), + ) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) + + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) + api.ShouldEventuallyMatch( + ctx, + kcp, + func(g Gomega) { + g.Expect(kcp.Spec.Replicas).To( + HaveValue(Equal(oldCPReplicas)), + "kcp replicas should not have changed", + ) + }, + ) } -func TestReconcileControlPlaneExternalEtcdUpgradeWithNoNamespace(t *testing.T) { +func TestReconcileControlPlaneExternalEtcdReadyControlPlaneUpgrade(t *testing.T) { g := NewWithT(t) c := env.Client() api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) - cp.Cluster.Spec.ManagedExternalEtcdRef.Namespace = "" + cp.EtcdCluster.Status.Ready = true + cp.EtcdCluster.Status.ObservedGeneration = 1 + + var oldCPReplicas int32 = 3 + var newCPReplicas int32 = 4 + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(oldCPReplicas) + clientutil.AddAnnotation(cp.KubeadmControlPlane, clusterv1.PausedAnnotation, "true") + // an existing kcp should already have etcd endpoints + cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{"https://1.1.1.1:2379"} envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + cp.KubeadmControlPlane.Spec.Replicas = ptr.Int32(newCPReplicas) + // by default providers code will generate kcp with empty endpoints, so we imitate that here + cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{} + + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To( + Equal(controller.Result{}), + ) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) + + kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) + api.ShouldEventuallyMatch( + ctx, + kcp, + func(g Gomega) { + g.Expect(kcp.Annotations).To( + HaveKey("cluster.x-k8s.io/skip-pause-cp-managed-etcd"), + "kcp should have skip pause annotation", + ) + g.Expect(annotations.HasPaused(kcp)).To( + BeFalse(), "kcp should not be paused", + ) + g.Expect(kcp.Spec.Replicas).To( + HaveValue(Equal(newCPReplicas)), + "kcp replicas should have been updated", + ) + g.Expect(kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints).To( + HaveExactElements("https://1.1.1.1:2379"), + "Etcd endpoints should remain the same and not be emptied", + ) + }, + ) } -func TestReconcileControlPlaneExternalEtcdWithExistingEndpoints(t *testing.T) { +func TestReconcileControlPlaneExternalEtcdUpgradeWithNoNamespace(t *testing.T) { g := NewWithT(t) c := env.Client() api := envtest.NewAPIExpecter(t, c) ctx := context.Background() ns := env.CreateNamespaceForTest(ctx, t) + log := test.NewNullLogger() cp := controlPlaneExternalEtcd(ns) - cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{"https://1.1.1.1:2379"} + cp.Cluster.Spec.ManagedExternalEtcdRef.Namespace = "" + cp.EtcdCluster.Status.Ready = true + cp.EtcdCluster.Status.ObservedGeneration = 1 envtest.CreateObjs(ctx, t, c, cp.AllObjects()...) - cp.KubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints = []string{} - g.Expect(clusters.ReconcileControlPlane(ctx, c, cp)).To(Equal(controller.Result{})) + g.Expect(clusters.ReconcileControlPlane(ctx, log, c, cp)).To(Equal(controller.Result{})) api.ShouldEventuallyExist(ctx, cp.Cluster) api.ShouldEventuallyExist(ctx, cp.KubeadmControlPlane) api.ShouldEventuallyExist(ctx, cp.ControlPlaneMachineTemplate) api.ShouldEventuallyExist(ctx, cp.ProviderCluster) api.ShouldEventuallyExist(ctx, cp.EtcdCluster) api.ShouldEventuallyExist(ctx, cp.EtcdMachineTemplate) - - kcp := envtest.CloneNameNamespace(cp.KubeadmControlPlane) - api.ShouldEventuallyMatch(ctx, kcp, func(g Gomega) { - endpoints := kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.Etcd.External.Endpoints - g.Expect(endpoints).To(ContainElement("https://1.1.1.1:2379")) - }) } func controlPlaneStackedEtcd(namespace string) *clusters.ControlPlane { diff --git a/pkg/providers/cloudstack/reconciler/reconciler.go b/pkg/providers/cloudstack/reconciler/reconciler.go index 7efbc954dc5a..f0c7ed564733 100644 --- a/pkg/providers/cloudstack/reconciler/reconciler.go +++ b/pkg/providers/cloudstack/reconciler/reconciler.go @@ -125,7 +125,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, &clusters.ControlPlane{ + return clusters.ReconcileControlPlane(ctx, log, r.client, &clusters.ControlPlane{ Cluster: cp.Cluster, ProviderCluster: cp.ProviderCluster, KubeadmControlPlane: cp.KubeadmControlPlane, diff --git a/pkg/providers/docker/reconciler/reconciler.go b/pkg/providers/docker/reconciler/reconciler.go index 810a16c6098f..75724e49f8ed 100644 --- a/pkg/providers/docker/reconciler/reconciler.go +++ b/pkg/providers/docker/reconciler/reconciler.go @@ -112,7 +112,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, if err != nil { return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, &clusters.ControlPlane{ + return clusters.ReconcileControlPlane(ctx, log, r.client, &clusters.ControlPlane{ Cluster: cp.Cluster, ProviderCluster: cp.ProviderCluster, KubeadmControlPlane: cp.KubeadmControlPlane, diff --git a/pkg/providers/nutanix/reconciler/reconciler.go b/pkg/providers/nutanix/reconciler/reconciler.go index 63b692773e53..830dbf73ff9a 100644 --- a/pkg/providers/nutanix/reconciler/reconciler.go +++ b/pkg/providers/nutanix/reconciler/reconciler.go @@ -194,7 +194,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, toClientControlPlane(cp)) + return clusters.ReconcileControlPlane(ctx, log, r.client, toClientControlPlane(cp)) } func toClientControlPlane(cp *nutanix.ControlPlane) *clusters.ControlPlane { diff --git a/pkg/providers/snow/reconciler/reconciler.go b/pkg/providers/snow/reconciler/reconciler.go index 0b938d05a630..495a5986ab36 100644 --- a/pkg/providers/snow/reconciler/reconciler.go +++ b/pkg/providers/snow/reconciler/reconciler.go @@ -111,7 +111,7 @@ func (s *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, s.client, toClientControlPlane(cp)) + return clusters.ReconcileControlPlane(ctx, log, s.client, toClientControlPlane(cp)) } func (r *Reconciler) CheckControlPlaneReady(ctx context.Context, log logr.Logger, clusterSpec *cluster.Spec) (controller.Result, error) { diff --git a/pkg/providers/tinkerbell/reconciler/reconciler.go b/pkg/providers/tinkerbell/reconciler/reconciler.go index d1264b69690f..7cd48064004d 100644 --- a/pkg/providers/tinkerbell/reconciler/reconciler.go +++ b/pkg/providers/tinkerbell/reconciler/reconciler.go @@ -235,7 +235,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, log = log.WithValues("phase", "reconcileControlPlane") log.Info("Applying control plane CAPI objects") - return clusters.ReconcileControlPlane(ctx, r.client, toClientControlPlane(tinkerbellScope.ControlPlane)) + return clusters.ReconcileControlPlane(ctx, log, r.client, toClientControlPlane(tinkerbellScope.ControlPlane)) } // CheckControlPlaneReady checks whether the control plane for an eks-a cluster is ready or not. diff --git a/pkg/providers/vsphere/reconciler/reconciler.go b/pkg/providers/vsphere/reconciler/reconciler.go index 1fe1fc624251..26c5676f50b8 100644 --- a/pkg/providers/vsphere/reconciler/reconciler.go +++ b/pkg/providers/vsphere/reconciler/reconciler.go @@ -192,7 +192,7 @@ func (r *Reconciler) ReconcileControlPlane(ctx context.Context, log logr.Logger, return controller.Result{}, err } - return clusters.ReconcileControlPlane(ctx, r.client, toClientControlPlane(cp)) + return clusters.ReconcileControlPlane(ctx, log, r.client, toClientControlPlane(cp)) } // CheckControlPlaneReady checks whether the control plane for an eks-a cluster is ready or not.