Add additional version check for cp ready (#7525)

pkg/awsiamauth/reconciler/reconciler_test.go

@@ -14,6 +14,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/utils/pointer"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -235,8 +236,10 @@ func TestReconcileRemoteGetClientError(t *testing.T) {
 			EksaVersion: &version,
 		},
 	}
+	kcpVersion := "test"
 	kcp := test.KubeadmControlPlane(func(kcp *controlplanev1.KubeadmControlPlane) {
 		kcp.Name = cluster.Name
+		kcp.Spec.Version = kcpVersion
 		kcp.Status = controlplanev1.KubeadmControlPlaneStatus{
 			Conditions: clusterv1.Conditions{
 				{
@@ -245,6 +248,7 @@ func TestReconcileRemoteGetClientError(t *testing.T) {
 					LastTransitionTime: metav1.NewTime(time.Now()),
 				},
 			},
+			Version: pointer.String(kcpVersion),
 		}
 	})
 	sec := &corev1.Secret{
@@ -309,8 +313,10 @@ func TestReconcileConfigMapNotFoundApplyError(t *testing.T) {
 			EksaVersion: &version,
 		},
 	}
+	kcpVersion := "test"
 	kcp := test.KubeadmControlPlane(func(kcp *controlplanev1.KubeadmControlPlane) {
 		kcp.Name = cluster.Name
+		kcp.Spec.Version = kcpVersion
 		kcp.Status = controlplanev1.KubeadmControlPlaneStatus{
 			Conditions: clusterv1.Conditions{
 				{
@@ -319,6 +325,7 @@ func TestReconcileConfigMapNotFoundApplyError(t *testing.T) {
 					LastTransitionTime: metav1.NewTime(time.Now()),
 				},
 			},
+			Version: pointer.String(kcpVersion),
 		}
 	})
 	sec := &corev1.Secret{

pkg/controller/clusters/clusterapi.go

@@ -34,7 +34,9 @@ func CheckControlPlaneReady(ctx context.Context, client client.Client, log logr.
 		return controller.ResultWithRequeue(5 * time.Second), nil
 	}
 
-	if !conditions.IsTrue(kcp, clusterapi.ReadyCondition) {
+	// Checking for version as well to avoid race condition of status not being updated in time at least for Kubernetes version upgrades
+	if !conditions.IsTrue(kcp, clusterapi.ReadyCondition) ||
+		kcp.Status.Version == nil || kcp.Spec.Version != *kcp.Status.Version {
 		log.Info("KCP is not ready yet, requeing")
 		return controller.ResultWithRequeue(30 * time.Second), nil
 	}

pkg/controller/clusters/clusterapi_test.go

@@ -8,6 +8,7 @@ import (
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
 	controllerruntime "sigs.k8s.io/controller-runtime"
@@ -26,13 +27,16 @@ func TestCheckControlPlaneReadyItIsReady(t *testing.T) {
 	g := NewWithT(t)
 	ctx := context.Background()
 	eksaCluster := eksaCluster()
+	kcpVersion := "test"
 	kcp := kcpObject(func(k *v1beta1.KubeadmControlPlane) {
+		k.Spec.Version = kcpVersion
 		k.Status.Conditions = clusterv1.Conditions{
 			{
 				Type:   clusterapi.ReadyCondition,
 				Status: corev1.ConditionTrue,
 			},
 		}
+		k.Status.Version = pointer.String(kcpVersion)
 	})
 
 	client := fake.NewClientBuilder().WithObjects(eksaCluster, kcp).Build()
@@ -74,7 +78,7 @@ func TestCheckControlPlaneNotReady(t *testing.T) {
 	)
 }
 
-func TestCheckControlPlaneStatusNotReady(t *testing.T) {
+func TestCheckControlPlaneReadyConditionStatusNotReady(t *testing.T) {
 	g := NewWithT(t)
 	ctx := context.Background()
 	eksaCluster := eksaCluster()
@@ -96,6 +100,51 @@ func TestCheckControlPlaneStatusNotReady(t *testing.T) {
 	)
 }
 
+func TestCheckControlPlaneVersionNilStatusNotReady(t *testing.T) {
+	g := NewWithT(t)
+	ctx := context.Background()
+	eksaCluster := eksaCluster()
+	kcp := kcpObject(func(k *v1beta1.KubeadmControlPlane) {
+		k.Status.Conditions = clusterv1.Conditions{
+			{
+				Type:   clusterapi.ReadyCondition,
+				Status: corev1.ConditionTrue,
+			},
+		}
+	})
+
+	client := fake.NewClientBuilder().WithObjects(eksaCluster, kcp).Build()
+
+	result, err := clusters.CheckControlPlaneReady(ctx, client, test.NewNullLogger(), eksaCluster)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(result).To(Equal(
+		controller.Result{Result: &controllerruntime.Result{RequeueAfter: 30 * time.Second}}),
+	)
+}
+
+func TestCheckControlPlaneVersionMismatchStatusNotReady(t *testing.T) {
+	g := NewWithT(t)
+	ctx := context.Background()
+	eksaCluster := eksaCluster()
+	kcp := kcpObject(func(k *v1beta1.KubeadmControlPlane) {
+		k.Status.Conditions = clusterv1.Conditions{
+			{
+				Type:   clusterapi.ReadyCondition,
+				Status: corev1.ConditionTrue,
+			},
+		}
+		k.Status.Version = pointer.String("test")
+	})
+
+	client := fake.NewClientBuilder().WithObjects(eksaCluster, kcp).Build()
+
+	result, err := clusters.CheckControlPlaneReady(ctx, client, test.NewNullLogger(), eksaCluster)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(result).To(Equal(
+		controller.Result{Result: &controllerruntime.Result{RequeueAfter: 30 * time.Second}}),
+	)
+}
+
 func eksaCluster() *anywherev1.Cluster {
 	return &anywherev1.Cluster{
 		TypeMeta: metav1.TypeMeta{

pkg/providers/snow/reconciler/reconciler_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/pointer"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
 	controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
@@ -230,6 +231,8 @@ func TestReconcilerReconcileControlPlane(t *testing.T) {
 
 func TestReconcilerCheckControlPlaneReadyItIsReady(t *testing.T) {
 	tt := newReconcilerTest(t)
+	kcpVersion := "test"
+	tt.kcp.Spec.Version = kcpVersion
 	tt.kcp.Status = controlplanev1.KubeadmControlPlaneStatus{
 		Conditions: clusterv1.Conditions{
 			{
@@ -238,6 +241,7 @@ func TestReconcilerCheckControlPlaneReadyItIsReady(t *testing.T) {
 				LastTransitionTime: metav1.NewTime(time.Now()),
 			},
 		},
+		Version: pointer.String(kcpVersion),
 	}
 	tt.eksaSupportObjs = append(tt.eksaSupportObjs, tt.kcp)
 	tt.withFakeClient()