From 6262c4253cbb99f3111c802530ff26b2efea7a79 Mon Sep 17 00:00:00 2001 From: Gong Zhang Date: Wed, 3 Jul 2024 17:30:55 +0800 Subject: [PATCH] Enhance E2E Tests to Filter Out Extra Provider-Specific Metadata Introduced a new field FilterMetadataBeforeValidation in E2E tests to allow filtering out labels and annotations of Machines, InfraMachines, BootstrapConfigs, and Nodes before validation. This feature enables the exclusion of additional infrastructure provider-specific labels that could otherwise cause test failures. Signed-off-by: Gong Zhang --- test/e2e/clusterclass_rollout.go | 86 +++++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/test/e2e/clusterclass_rollout.go b/test/e2e/clusterclass_rollout.go index e9ca19ea4581..4c6c3a9c2857 100644 --- a/test/e2e/clusterclass_rollout.go +++ b/test/e2e/clusterclass_rollout.go @@ -75,6 +75,12 @@ type ClusterClassRolloutSpecInput struct { // Allows to inject a function to be run after test namespace is created. // If not specified, this is a no-op. PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string) + + // FilterMetadataBeforeValidation allows filtering out labels and annotations of Machines, InfraMachines, + // BootstrapConfigs and Nodes before we validate them. + // This can be e.g. used to filter out additional infrastructure provider specific labels that would + // otherwise lead to a failed test. + FilterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta } // ClusterClassRolloutSpec implements a test that verifies the ClusterClass rollout behavior. @@ -142,7 +148,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"), }, clusterResources) - assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass) + assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation) By("Rolling out changes to control plane, MachineDeployments, and MachinePools (in-place)") machinesBeforeUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster) @@ -218,7 +224,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster) g.Expect(machinesAfterUpgrade.Equal(machinesBeforeUpgrade)).To(BeTrue(), "Machines must not be replaced through in-place rollout") }, 30*time.Second, 1*time.Second).Should(Succeed()) - assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass) + assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation) By("Rolling out changes to control plane, MachineDeployments, and MachinePools (rollout)") machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster) @@ -259,7 +265,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster) g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout") }, input.E2EConfig.GetIntervals(specName, "wait-control-plane")...).Should(Succeed()) - assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass) + assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation) By("Rolling out control plane and MachineDeployment (rolloutAfter)") machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster) @@ -287,7 +293,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster) g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout with rolloutAfter") }, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")...).Should(Succeed()) - assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass) + assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation) By("PASSED!") }) @@ -299,7 +305,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas } // assertClusterObjects asserts cluster objects by checking that all objects have the right labels, annotations and selectors. -func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) { +func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) { By("Checking cluster objects have the right labels, annotations and selectors") Eventually(func(g Gomega) { @@ -312,12 +318,12 @@ func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterPro // ControlPlane assertControlPlane(g, clusterClassObjects, clusterObjects, cluster, clusterClass) - assertControlPlaneMachines(g, clusterObjects, cluster) + assertControlPlaneMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation) // MachineDeployments assertMachineDeployments(g, clusterClassObjects, clusterObjects, cluster, clusterClass) assertMachineSets(g, clusterObjects, cluster) - assertMachineSetsMachines(g, clusterObjects, cluster) + assertMachineSetsMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation) // MachinePools assertMachinePools(g, clusterClassObjects, clusterObjects, cluster, clusterClass) @@ -430,13 +436,17 @@ func assertControlPlane(g Gomega, clusterClassObjects clusterClassObjects, clust )) } -func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) { +func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) { controlPlaneMachineTemplateMetadata := mustMetadata(contract.ControlPlane().MachineTemplate().Metadata().Get(clusterObjects.ControlPlane)) controlPlaneInfrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(clusterObjects.ControlPlaneInfrastructureMachineTemplate)) for _, machine := range clusterObjects.ControlPlaneMachines { // ControlPlane Machine.metadata - g.Expect(machine.Labels).To(BeEquivalentTo( + machineMetadata := clusterv1.ObjectMeta{Labels: machine.Labels, Annotations: machine.Annotations} + if filterMetadataBeforeValidation != nil { + machineMetadata = filterMetadataBeforeValidation(machine) + } + g.Expect(machineMetadata.Labels).To(BeEquivalentTo( union( map[string]string{ clusterv1.ClusterNameLabel: cluster.Name, @@ -457,9 +467,13 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster // ControlPlane Machine InfrastructureMachine.metadata infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name] + infrastructureMachineMetadata := clusterv1.ObjectMeta{Labels: infrastructureMachine.GetLabels(), Annotations: infrastructureMachine.GetAnnotations()} + if filterMetadataBeforeValidation != nil { + infrastructureMachineMetadata = filterMetadataBeforeValidation(infrastructureMachine) + } controlPlaneMachineTemplateInfrastructureRef, err := contract.ControlPlane().MachineTemplate().InfrastructureRef().Get(clusterObjects.ControlPlane) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo( + g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo( union( map[string]string{ clusterv1.ClusterNameLabel: cluster.Name, @@ -471,7 +485,7 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster controlPlaneInfrastructureMachineTemplateTemplateMetadata.Labels, ), )) - g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo( + g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo( union( map[string]string{ clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(controlPlaneMachineTemplateInfrastructureRef), @@ -484,7 +498,11 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster // ControlPlane Machine BootstrapConfig.metadata bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name] - g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo( + bootstrapConfigMetadata := clusterv1.ObjectMeta{Labels: bootstrapConfig.GetLabels(), Annotations: bootstrapConfig.GetAnnotations()} + if filterMetadataBeforeValidation != nil { + bootstrapConfigMetadata = filterMetadataBeforeValidation(bootstrapConfig) + } + g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo( union( map[string]string{ clusterv1.ClusterNameLabel: cluster.Name, @@ -497,7 +515,7 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster )) g.Expect( union( - bootstrapConfig.GetAnnotations(), + bootstrapConfigMetadata.Annotations, ).without(g, clusterv1.MachineCertificatesExpiryDateAnnotation), ).To(BeEquivalentTo( controlPlaneMachineTemplateMetadata.Annotations, @@ -505,8 +523,12 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster // ControlPlane Machine Node.metadata node := clusterObjects.NodesByMachine[machine.Name] - for k, v := range getManagedLabels(machine.Labels) { - g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v)) + nodeMetadata := clusterv1.ObjectMeta{Labels: node.GetLabels(), Annotations: node.GetAnnotations()} + if filterMetadataBeforeValidation != nil { + nodeMetadata = filterMetadataBeforeValidation(node) + } + for k, v := range getManagedLabels(machineMetadata.Labels) { + g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v)) } } } @@ -790,7 +812,7 @@ func assertMachineSets(g Gomega, clusterObjects clusterObjects, cluster *cluster } } -func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) { +func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) { for _, machineDeployment := range clusterObjects.MachineDeployments { mdTopology := getMDTopology(cluster, machineDeployment) infrastructureMachineTemplate := clusterObjects.InfrastructureMachineTemplateByMachineDeployment[machineDeployment.Name] @@ -802,8 +824,12 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster machineTemplateHash := machineSet.Labels[clusterv1.MachineDeploymentUniqueLabel] for _, machine := range clusterObjects.MachinesByMachineSet[machineSet.Name] { + machineMetadata := clusterv1.ObjectMeta{Labels: machine.Labels, Annotations: machine.Annotations} + if filterMetadataBeforeValidation != nil { + machineMetadata = filterMetadataBeforeValidation(machine) + } // MachineDeployment MachineSet Machine.metadata - g.Expect(machine.Labels).To(BeEquivalentTo( + g.Expect(machineMetadata.Labels).To(BeEquivalentTo( union( map[string]string{ clusterv1.ClusterNameLabel: cluster.Name, @@ -816,13 +842,17 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster machineSet.Spec.Template.Labels, ), )) - g.Expect(machine.Annotations).To(BeEquivalentTo( + g.Expect(machineMetadata.Annotations).To(BeEquivalentTo( machineSet.Spec.Template.Annotations, )) // MachineDeployment MachineSet Machine InfrastructureMachine.metadata infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name] - g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo( + infrastructureMachineMetadata := clusterv1.ObjectMeta{Labels: infrastructureMachine.GetLabels(), Annotations: infrastructureMachine.GetAnnotations()} + if filterMetadataBeforeValidation != nil { + infrastructureMachineMetadata = filterMetadataBeforeValidation(infrastructureMachine) + } + g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo( union( map[string]string{ clusterv1.ClusterNameLabel: cluster.Name, @@ -836,7 +866,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster infrastructureMachineTemplateTemplateMetadata.Labels, ), )) - g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo( + g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo( union( map[string]string{ clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(&machineSet.Spec.Template.Spec.InfrastructureRef), @@ -849,7 +879,11 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster // MachineDeployment MachineSet Machine BootstrapConfig.metadata bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name] - g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo( + bootstrapConfigMetadata := clusterv1.ObjectMeta{Labels: bootstrapConfig.GetLabels(), Annotations: bootstrapConfig.GetAnnotations()} + if filterMetadataBeforeValidation != nil { + bootstrapConfigMetadata = filterMetadataBeforeValidation(bootstrapConfig) + } + g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo( union( map[string]string{ clusterv1.ClusterNameLabel: cluster.Name, @@ -863,7 +897,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster bootstrapConfigTemplateTemplateMetadata.Labels, ), )) - g.Expect(bootstrapConfig.GetAnnotations()).To(BeEquivalentTo( + g.Expect(bootstrapConfigMetadata.Annotations).To(BeEquivalentTo( union( map[string]string{ clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(machineSet.Spec.Template.Spec.Bootstrap.ConfigRef), @@ -876,8 +910,12 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster // MachineDeployment MachineSet Machine Node.metadata node := clusterObjects.NodesByMachine[machine.Name] - for k, v := range getManagedLabels(machine.Labels) { - g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v)) + nodeMetadata := clusterv1.ObjectMeta{Labels: node.GetLabels(), Annotations: node.GetAnnotations()} + if filterMetadataBeforeValidation != nil { + nodeMetadata = filterMetadataBeforeValidation(node) + } + for k, v := range getManagedLabels(machineMetadata.Labels) { + g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v)) } } }