tests: Add test suite common resource labels and fix dnsrecord cleanup (

#941)

* tests: Add common label to all test suite resources

All resources created with the common helpers in the integration test
suite have a comman label added to more easily find resources created by
the test suite:

```
kubectl get ns,gatewayclass,gateway,dnspolicy -l app.kubernetes.io/part-of=kuadrant-test-suite -A
```

* tests: Ensure all dnsrecords are removed in cleanup

Ensures all DNSrecord resources in the current spec test namespace are
marked for deletion and removed before removing the secret.

* tests: Allow overriding packages for integration test

Adds a new variable `INTEGRATION_TEST_PACKAGES` which sets the packages
used by the `test-integration` make target allowing it to be more easily
modified for local development testing.

Example:
```
INTEGRATION_TEST_PACKAGES=tests/common/dnspolicy/... INTEGRATION_TESTS_EXTRA_ARGS='-v --repeat=4 --focus="valid target"' INTEGRATION_TEST_NUM_PROCESSES=4 INTEGRATION_TEST_NUM_CORES=2 make test-integration
```
---------

Signed-off-by: Michael Nairn <[email protected]>

make/integration-tests.mk

@@ -2,6 +2,7 @@ INTEGRATION_COVER_PKGS = ./pkg/...,./controllers/...,./api/...
 INTEGRATION_TESTS_EXTRA_ARGS ?=
 INTEGRATION_TEST_NUM_CORES ?= 4
 INTEGRATION_TEST_NUM_PROCESSES ?= 10
+INTEGRATION_TEST_PACKAGES ?= tests/common/...
 
 ##@ Integration tests
 
@@ -92,4 +93,4 @@ test-integration: clean-cov generate fmt vet ginkgo ## Requires kubernetes clust
 		--fail-on-pending \
 		--keep-going \
 		--trace \
-		$(INTEGRATION_TESTS_EXTRA_ARGS) tests/common/...
+		$(INTEGRATION_TESTS_EXTRA_ARGS) $(INTEGRATION_TEST_PACKAGES)

tests/common/dnspolicy/dnspolicy_controller_single_cluster_test.go

@@ -115,15 +115,24 @@ var _ = Describe("DNSPolicy Single Cluster", func() {
 		if dnsPolicy != nil {
 			err := k8sClient.Delete(ctx, dnsPolicy)
 			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
-			// Wait until dns records are finished deleting since it can't finish deleting without the DNS provider secret
-			Eventually(func(g Gomega) {
-				dnsRecords := &kuadrantdnsv1alpha1.DNSRecordList{}
-				err := k8sClient.List(ctx, dnsRecords, client.InNamespace(testNamespace))
-				g.Expect(err).NotTo(HaveOccurred())
-				g.Expect(dnsRecords.Items).To(HaveLen(0))
-			}).WithContext(ctx).Should(Succeed())
+		}
 
+		//Ensure all dns records in the test namespace are deleted
+		dnsRecords := &kuadrantdnsv1alpha1.DNSRecordList{}
+		err := k8sClient.List(ctx, dnsRecords, client.InNamespace(testNamespace))
+		Expect(err).ToNot(HaveOccurred())
+		for _, record := range dnsRecords.Items {
+			err := k8sClient.Delete(ctx, &record, client.PropagationPolicy(metav1.DeletePropagationForeground))
+			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
 		}
+
+		// Wait until dns records are finished deleting since it can't finish deleting without the DNS provider secret
+		Eventually(func(g Gomega) {
+			err := k8sClient.List(ctx, dnsRecords, client.InNamespace(testNamespace))
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(dnsRecords.Items).To(HaveLen(0))
+		}).WithContext(ctx).Should(Succeed())
+
 		if dnsProviderSecret != nil {
 			err := k8sClient.Delete(ctx, dnsProviderSecret)
 			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
@@ -138,7 +147,7 @@ var _ = Describe("DNSPolicy Single Cluster", func() {
 	Context("simple routing strategy", func() {
 
 		BeforeEach(func(ctx SpecContext) {
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway(tests.GatewayName)
 			Expect(k8sClient.Create(ctx, dnsPolicy)).To(Succeed())
@@ -197,7 +206,7 @@ var _ = Describe("DNSPolicy Single Cluster", func() {
 
 		Context("with default geo", func() {
 			BeforeEach(func(ctx SpecContext) {
-				dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+				dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 					WithProviderSecret(*dnsProviderSecret).
 					WithTargetGateway(tests.GatewayName).
 					WithLoadBalancingFor(120, "IE", true)
@@ -317,7 +326,7 @@ var _ = Describe("DNSPolicy Single Cluster", func() {
 
 		Context("without default geo", func() {
 			BeforeEach(func(ctx SpecContext) {
-				dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+				dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 					WithProviderSecret(*dnsProviderSecret).
 					WithTargetGateway(tests.GatewayName).
 					WithLoadBalancingFor(120, "IE", false)

tests/common/dnspolicy/dnspolicy_controller_test.go

@@ -59,14 +59,24 @@ var _ = Describe("DNSPolicy controller", func() {
 		if dnsPolicy != nil {
 			err := k8sClient.Delete(ctx, dnsPolicy)
 			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
-			// Wait until dns records are finished deleting since it can't finish deleting without the DNS provider secret
-			Eventually(func(g Gomega) {
-				dnsRecords := &kuadrantdnsv1alpha1.DNSRecordList{}
-				err := k8sClient.List(ctx, dnsRecords, client.InNamespace(testNamespace))
-				g.Expect(err).NotTo(HaveOccurred())
-				g.Expect(dnsRecords.Items).To(HaveLen(0))
-			}).WithContext(ctx).Should(Succeed())
 		}
+
+		//Ensure all dns records in the test namespace are deleted
+		dnsRecords := &kuadrantdnsv1alpha1.DNSRecordList{}
+		err := k8sClient.List(ctx, dnsRecords, client.InNamespace(testNamespace))
+		Expect(err).ToNot(HaveOccurred())
+		for _, record := range dnsRecords.Items {
+			err := k8sClient.Delete(ctx, &record, client.PropagationPolicy(metav1.DeletePropagationForeground))
+			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
+		}
+
+		// Wait until dns records are finished deleting since it can't finish deleting without the DNS provider secret
+		Eventually(func(g Gomega) {
+			err := k8sClient.List(ctx, dnsRecords, client.InNamespace(testNamespace))
+			g.Expect(err).NotTo(HaveOccurred())
+			g.Expect(dnsRecords.Items).To(HaveLen(0))
+		}).WithContext(ctx).Should(Succeed())
+
 		if dnsProviderSecret != nil {
 			err := k8sClient.Delete(ctx, dnsProviderSecret)
 			Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred())
@@ -84,7 +94,7 @@ var _ = Describe("DNSPolicy controller", func() {
 			WithHTTPListener(tests.ListenerNameOne, tests.HostTwo(domain)).Gateway
 
 		// simple should succeed
-		dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+		dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 			WithProviderSecret(*dnsProviderSecret).
 			WithTargetGateway("test-gateway")
 		Expect(k8sClient.Create(ctx, dnsPolicy)).To(Succeed())
@@ -105,7 +115,7 @@ var _ = Describe("DNSPolicy controller", func() {
 		Expect(k8sClient.Delete(ctx, dnsPolicy)).ToNot(HaveOccurred())
 
 		// loadbalanced should succeed
-		dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+		dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 			WithProviderSecret(*dnsProviderSecret).
 			WithTargetGateway("test-gateway").
 			WithLoadBalancingFor(100, "foo", false)
@@ -145,12 +155,12 @@ var _ = Describe("DNSPolicy controller", func() {
 			WithHTTPListener(tests.ListenerNameOne, tests.HostTwo(domain)).Gateway
 
 		// should not allow an empty providerRef list
-		dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+		dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 			WithTargetGateway("test-gateway")
 		Expect(k8sClient.Create(ctx, dnsPolicy)).To(MatchError(ContainSubstring("spec.providerRefs: Required value")))
 
 		// should create with a single providerRef
-		dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+		dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 			WithProviderSecret(*dnsProviderSecret).
 			WithTargetGateway("test-gateway")
 		Expect(k8sClient.Create(ctx, dnsPolicy)).To(Succeed())
@@ -217,7 +227,7 @@ var _ = Describe("DNSPolicy controller", func() {
 		}, tests.TimeoutMedium, tests.RetryIntervalMedium).Should(Succeed())
 
 		// Create policy1 targeting gateway1 with simple routing strategy
-		dnsPolicy1 := v1alpha1.NewDNSPolicy("test-dns-policy1", testNamespace).
+		dnsPolicy1 := tests.NewDNSPolicy("test-dns-policy1", testNamespace).
 			WithProviderSecret(*dnsProviderSecret).
 			WithTargetGateway("test-gateway1")
 		Expect(k8sClient.Create(ctx, dnsPolicy1)).To(Succeed())
@@ -265,7 +275,7 @@ var _ = Describe("DNSPolicy controller", func() {
 		}, tests.TimeoutLong, tests.RetryIntervalMedium).Should(Succeed())
 
 		// create policy2 targeting gateway2 with the load-balanced strategy
-		dnsPolicy2 := v1alpha1.NewDNSPolicy("test-dns-policy2", testNamespace).
+		dnsPolicy2 := tests.NewDNSPolicy("test-dns-policy2", testNamespace).
 			WithProviderSecret(*dnsProviderSecret).
 			WithTargetGateway("test-gateway2").
 			WithLoadBalancingFor(100, "foo", false)
@@ -336,7 +346,7 @@ var _ = Describe("DNSPolicy controller", func() {
 
 	Context("invalid target", func() {
 		It("should have accepted condition with status false and correct reason", func(ctx SpecContext) {
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway("test-gateway")
 			Expect(k8sClient.Create(ctx, dnsPolicy)).To(Succeed())
@@ -411,13 +421,13 @@ var _ = Describe("DNSPolicy controller", func() {
 			}, tests.TimeoutMedium, tests.RetryIntervalMedium).Should(Succeed())
 
 			// Create policy1 targeting gateway1 with simple routing strategy
-			dnsPolicy1 := v1alpha1.NewDNSPolicy("test-dns-policy1", testNamespace).
+			dnsPolicy1 := tests.NewDNSPolicy("test-dns-policy1", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway("test-gateway1")
 			Expect(k8sClient.Create(ctx, dnsPolicy1)).To(Succeed())
 
 			// create policy2 targeting gateway2 with the load-balanced strategy
-			dnsPolicy2 := v1alpha1.NewDNSPolicy("test-dns-policy2", testNamespace).
+			dnsPolicy2 := tests.NewDNSPolicy("test-dns-policy2", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway("test-gateway2").
 				WithLoadBalancingFor(100, "foo", false)
@@ -464,7 +474,7 @@ var _ = Describe("DNSPolicy controller", func() {
 			gateway = tests.NewGatewayBuilder(testGatewayName, gatewayClass.Name, testNamespace).
 				WithHTTPListener(tests.ListenerNameOne, tests.HostTwo(domain)).
 				Gateway
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway(testGatewayName)
 
@@ -512,7 +522,7 @@ var _ = Describe("DNSPolicy controller", func() {
 				WithHTTPListener(tests.ListenerNameOne, tests.HostOne(domain)).
 				WithHTTPListener(tests.ListenerNameWildcard, tests.HostWildcard(domain)).
 				Gateway
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway(tests.GatewayName)
 
@@ -802,7 +812,7 @@ var _ = Describe("DNSPolicy controller", func() {
 
 	Context("cel validation", func() {
 		It("should error targeting invalid group", func(ctx SpecContext) {
-			p := v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			p := tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway("gateway")
 			p.Spec.TargetRef.Group = "not-gateway.networking.k8s.io"
@@ -813,7 +823,7 @@ var _ = Describe("DNSPolicy controller", func() {
 		}, testTimeOut)
 
 		It("should error targeting invalid kind", func(ctx SpecContext) {
-			p := v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			p := tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway("gateway")
 			p.Spec.TargetRef.Kind = "TCPRoute"
@@ -830,7 +840,7 @@ var _ = Describe("DNSPolicy controller", func() {
 				WithHTTPListener(tests.ListenerNameOne, tests.HostOne(domain)).
 				WithHTTPListener(tests.ListenerNameWildcard, tests.HostWildcard(domain)).
 				Gateway
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway(tests.GatewayName)
 			Expect(k8sClient.Create(ctx, gateway)).To(Succeed())
@@ -866,7 +876,7 @@ var _ = Describe("DNSPolicy controller", func() {
 
 		})
 
-		It("should have an accpeterd and enforced policy with additional context", func(ctx SpecContext) {
+		It("should have an accepted and enforced policy with additional context", func(ctx SpecContext) {
 			Eventually(func(g Gomega) {
 				err := k8sClient.Get(ctx, client.ObjectKeyFromObject(dnsPolicy), dnsPolicy)
 				g.Expect(err).NotTo(HaveOccurred())
@@ -899,7 +909,7 @@ var _ = Describe("DNSPolicy controller", func() {
 			Expect(k8sClient.Create(ctx, gateway)).To(Succeed())
 		})
 		It("should create a DNSPolicy with an invalid CIDR", func(ctx SpecContext) {
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway(gateway.Name).
 				WithExcludeAddresses([]string{"1.1.1.1/345"})
@@ -948,7 +958,7 @@ var _ = Describe("DNSPolicy controller", func() {
 		})
 
 		It("should create a DNSPolicy valid exclude addresses", func(ctx SpecContext) {
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway(gateway.Name).
 				WithExcludeAddresses([]string{tests.IPAddressOne})
@@ -1022,7 +1032,7 @@ var _ = Describe("DNSPolicy controller", func() {
 
 		})
 		It("should not create a DNSRecords if no endpoints due to DNSPolicy exclude addresses", func(ctx SpecContext) {
-			dnsPolicy = v1alpha1.NewDNSPolicy("test-dns-policy", testNamespace).
+			dnsPolicy = tests.NewDNSPolicy("test-dns-policy", testNamespace).
 				WithProviderSecret(*dnsProviderSecret).
 				WithTargetGateway(gateway.Name).
 				WithExcludeAddresses([]string{tests.IPAddressOne, tests.IPAddressTwo})

tests/commons.go

@@ -31,6 +31,7 @@ import (
 	kuadrantdnsv1alpha1 "github.com/kuadrant/dns-operator/api/v1alpha1"
 	kuadrantdnsbuilder "github.com/kuadrant/dns-operator/pkg/builder"
 
+	kuadrantv1alpha1 "github.com/kuadrant/kuadrant-operator/api/v1alpha1"
 	kuadrantv1beta1 "github.com/kuadrant/kuadrant-operator/api/v1beta1"
 	kuadrantv1beta2 "github.com/kuadrant/kuadrant-operator/api/v1beta2"
 	kuadrantv1beta3 "github.com/kuadrant/kuadrant-operator/api/v1beta3"
@@ -54,6 +55,8 @@ const (
 	HTTPRouteName        = "toystore-route"
 )
 
+var CommonLabels = map[string]string{"app.kubernetes.io/part-of": "kuadrant-test-suite"}
+
 var GatewayClassName string
 
 func HostWildcard(domain string) string {
@@ -77,7 +80,7 @@ func BuildBasicGateway(gwName, ns string, mutateFns ...func(*gatewayapiv1.Gatewa
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        gwName,
 			Namespace:   ns,
-			Labels:      map[string]string{"app": "rlptest"},
+			Labels:      CommonLabels,
 			Annotations: map[string]string{"networking.istio.io/service-type": string(corev1.ServiceTypeClusterIP)},
 		},
 		Spec: gatewayapiv1.GatewaySpec{
@@ -109,7 +112,7 @@ func DeleteNamespace(ctx context.Context, cl client.Client, namespace string) {
 func CreateNamespace(ctx context.Context, cl client.Client) string {
 	nsObject := &corev1.Namespace{
 		TypeMeta:   metav1.TypeMeta{APIVersion: "v1", Kind: "Namespace"},
-		ObjectMeta: metav1.ObjectMeta{GenerateName: "test-namespace-"},
+		ObjectMeta: metav1.ObjectMeta{GenerateName: "test-namespace-", Labels: CommonLabels},
 	}
 	Expect(cl.Create(ctx, nsObject)).To(Succeed())
 
@@ -129,6 +132,7 @@ func ApplyKuadrantCRWithName(ctx context.Context, cl client.Client, namespace, n
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
+			Labels:    CommonLabels,
 		},
 	}
 
@@ -167,7 +171,7 @@ func BuildBasicHttpRoute(routeName, gwName, ns string, hostnames []string, mutat
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      routeName,
 			Namespace: ns,
-			Labels:    map[string]string{"app": "rlptest"},
+			Labels:    CommonLabels,
 		},
 		Spec: gatewayapiv1.HTTPRouteSpec{
 			CommonRouteSpec: gatewayapiv1.CommonRouteSpec{
@@ -458,10 +462,18 @@ func ObjectDoesNotExist(k8sClient client.Client, obj client.Object) func() bool
 // DNS
 
 func BuildInMemoryCredentialsSecret(name, ns, initDomain string) *corev1.Secret {
-	return kuadrantdnsbuilder.NewProviderBuilder(name, ns).
+	secret := kuadrantdnsbuilder.NewProviderBuilder(name, ns).
 		For(kuadrantdnsv1alpha1.SecretTypeKuadrantInmemory).
 		WithZonesInitialisedFor(initDomain).
 		Build()
+	secret.Labels = CommonLabels
+	return secret
+}
+
+func NewDNSPolicy(name, ns string) *kuadrantv1alpha1.DNSPolicy {
+	p := kuadrantv1alpha1.NewDNSPolicy(name, ns)
+	p.Labels = CommonLabels
+	return p
 }
 
 // EndpointsTraversable consumes an array of endpoints and returns a boolean
@@ -501,6 +513,7 @@ func BuildGatewayClass(name, ns, controllerName string) *gatewayapiv1.GatewayCla
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: ns,
+			Labels:    CommonLabels,
 		},
 		Spec: gatewayapiv1.GatewayClassSpec{
 			ControllerName: gatewayapiv1.GatewayController(controllerName),
@@ -519,6 +532,7 @@ func NewGatewayBuilder(gwName, gwClassName, ns string) *GatewayBuilder {
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      gwName,
 				Namespace: ns,
+				Labels:    CommonLabels,
 			},
 			Spec: gatewayapiv1.GatewaySpec{
 				GatewayClassName: gatewayapiv1.ObjectName(gwClassName),
@@ -589,6 +603,7 @@ func BuildSelfSignedIssuer(name, ns string) (*certmanv1.Issuer, *certmanmetav1.O
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: ns,
+			Labels:    CommonLabels,
 		},
 		Spec: createSelfSignedIssuerSpec(),
 	}
@@ -605,6 +620,7 @@ func BuildSelfSignedClusterIssuer(name, ns string) (*certmanv1.ClusterIssuer, *c
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: ns,
+			Labels:    CommonLabels,
 		},
 		Spec: createSelfSignedIssuerSpec(),
 	}