From 78606f85c7ad2f44063ef250f4483f752d6e883f Mon Sep 17 00:00:00 2001
From: Christian Schlotter <christian.schlotter@broadcom.com>
Date: Thu, 25 Jul 2024 14:02:09 +0200
Subject: [PATCH] Fix flake where VSphereFailureDomain still exists from
 previous test

---
 controllers/vspherecluster_reconciler_test.go |  2 +-
 internal/test/helpers/envtest.go              | 46 +++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/controllers/vspherecluster_reconciler_test.go b/controllers/vspherecluster_reconciler_test.go
index 797c2a1fc1..ca7a245d35 100644
--- a/controllers/vspherecluster_reconciler_test.go
+++ b/controllers/vspherecluster_reconciler_test.go
@@ -358,7 +358,7 @@ var _ = Describe("VIM based VSphere ClusterReconciler", func() {
 		})
 
 		AfterEach(func() {
-			Expect(testEnv.Cleanup(ctx, namespace, capiCluster, instance, zoneOne)).To(Succeed())
+			Expect(testEnv.CleanupAndWait(ctx, instance, zoneOne, capiCluster, namespace)).To(Succeed())
 		})
 
 		It("should reconcile a cluster", func() {
diff --git a/internal/test/helpers/envtest.go b/internal/test/helpers/envtest.go
index 584e80974f..e75d6faba1 100644
--- a/internal/test/helpers/envtest.go
+++ b/internal/test/helpers/envtest.go
@@ -25,8 +25,10 @@ import (
 	"path/filepath"
 	goruntime "runtime"
 	"strings"
+	"time"
 
 	"github.com/onsi/ginkgo/v2"
+	"github.com/pkg/errors"
 	"github.com/vmware/govmomi/simulator"
 	"golang.org/x/tools/go/packages"
 	admissionv1 "k8s.io/api/admissionregistration/v1"
@@ -37,6 +39,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
@@ -99,6 +102,15 @@ func init() {
 	}
 }
 
+var (
+	cacheSyncBackoff = wait.Backoff{
+		Duration: 100 * time.Millisecond,
+		Factor:   1.5,
+		Steps:    8,
+		Jitter:   0.4,
+	}
+)
+
 type (
 	// TestEnvironment encapsulates a Kubernetes local test environment.
 	TestEnvironment struct {
@@ -233,6 +245,40 @@ func (t *TestEnvironment) Cleanup(ctx context.Context, objs ...client.Object) er
 	return kerrors.NewAggregate(errs)
 }
 
+// CleanupAndWait removes objects from the TestEnvironment and waits for them to be gone.
+func (t *TestEnvironment) CleanupAndWait(ctx context.Context, objs ...client.Object) error {
+	if err := t.Cleanup(ctx, objs...); err != nil {
+		return err
+	}
+
+	// Makes sure the cache is updated with the deleted object
+	errs := []error{}
+	for _, o := range objs {
+		// Ignoring namespaces because in testenv the namespace cleaner is not running.
+		if _, ok := o.(*corev1.Namespace); ok {
+			continue
+		}
+
+		oCopy := o.DeepCopyObject().(client.Object)
+		key := client.ObjectKeyFromObject(o)
+		err := wait.ExponentialBackoff(
+			cacheSyncBackoff,
+			func() (done bool, err error) {
+				if err := t.Get(ctx, key, oCopy); err != nil {
+					if apierrors.IsNotFound(err) {
+						return true, nil
+					}
+					return false, err
+				}
+				return false, nil
+			})
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "key %s, %s is not being deleted from the testenv client cache", o.GetObjectKind().GroupVersionKind().String(), key))
+		}
+	}
+	return kerrors.NewAggregate(errs)
+}
+
 // CreateNamespace creates a new namespace in the TestEnvironment.
 func (t *TestEnvironment) CreateNamespace(ctx context.Context, generateName string) (*corev1.Namespace, error) {
 	ns := &corev1.Namespace{