Skip to content

Commit

Permalink
Enhance E2E Tests to Filter Out Extra Provider-Specific Metadata
Browse files Browse the repository at this point in the history 
Introduced a new field FilterMetadataBeforeValidation in E2E tests
to allow filtering out labels and annotations of Machines,
InfraMachines, BootstrapConfigs, and Nodes before validation.
This feature enables the exclusion of additional infrastructure
provider-specific labels that could otherwise cause test failures.

Signed-off-by: Gong Zhang <[email protected]>
  • Loading branch information
@zhanggbj
zhanggbj committed Jul 3, 2024
1 parent f57b8c8 commit 6262c42
Showing 1 changed file with 62 additions and 24 deletions.
86 changes: 62 additions & 24 deletions test/e2e/clusterclass_rollout.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@ type ClusterClassRolloutSpecInput struct {
// Allows to inject a function to be run after test namespace is created.
// If not specified, this is a no-op.
PostNamespaceCreated func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace string)

// FilterMetadataBeforeValidation allows filtering out labels and annotations of Machines, InfraMachines,
// BootstrapConfigs and Nodes before we validate them.
// This can be e.g. used to filter out additional infrastructure provider specific labels that would
// otherwise lead to a failed test.
FilterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta
}

// ClusterClassRolloutSpec implements a test that verifies the ClusterClass rollout behavior.
Expand Down Expand Up @@ -142,7 +148,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
WaitForMachinePools: input.E2EConfig.GetIntervals(specName, "wait-machine-pool-nodes"),
}, clusterResources)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out changes to control plane, MachineDeployments, and MachinePools (in-place)")
machinesBeforeUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -218,7 +224,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.Equal(machinesBeforeUpgrade)).To(BeTrue(), "Machines must not be replaced through in-place rollout")
}, 30*time.Second, 1*time.Second).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out changes to control plane, MachineDeployments, and MachinePools (rollout)")
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -259,7 +265,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout")
}, input.E2EConfig.GetIntervals(specName, "wait-control-plane")...).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("Rolling out control plane and MachineDeployment (rolloutAfter)")
machinesBeforeUpgrade = getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
Expand Down Expand Up @@ -287,7 +293,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
machinesAfterUpgrade := getMachinesByCluster(ctx, input.BootstrapClusterProxy.GetClient(), clusterResources.Cluster)
g.Expect(machinesAfterUpgrade.HasAny(machinesBeforeUpgrade.UnsortedList()...)).To(BeFalse(), "All Machines must be replaced through rollout with rolloutAfter")
}, input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade")...).Should(Succeed())
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass)
assertClusterObjects(ctx, input.BootstrapClusterProxy, clusterResources.Cluster, clusterResources.ClusterClass, input.FilterMetadataBeforeValidation)

By("PASSED!")
})
Expand All @@ -299,7 +305,7 @@ func ClusterClassRolloutSpec(ctx context.Context, inputGetter func() ClusterClas
}

// assertClusterObjects asserts cluster objects by checking that all objects have the right labels, annotations and selectors.
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass) {
func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterProxy, cluster *clusterv1.Cluster, clusterClass *clusterv1.ClusterClass, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
By("Checking cluster objects have the right labels, annotations and selectors")

Eventually(func(g Gomega) {
Expand All @@ -312,12 +318,12 @@ func assertClusterObjects(ctx context.Context, clusterProxy framework.ClusterPro

// ControlPlane
assertControlPlane(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
assertControlPlaneMachines(g, clusterObjects, cluster)
assertControlPlaneMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)

// MachineDeployments
assertMachineDeployments(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
assertMachineSets(g, clusterObjects, cluster)
assertMachineSetsMachines(g, clusterObjects, cluster)
assertMachineSetsMachines(g, clusterObjects, cluster, filterMetadataBeforeValidation)

// MachinePools
assertMachinePools(g, clusterClassObjects, clusterObjects, cluster, clusterClass)
Expand Down Expand Up @@ -430,13 +436,17 @@ func assertControlPlane(g Gomega, clusterClassObjects clusterClassObjects, clust
))
}

func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
controlPlaneMachineTemplateMetadata := mustMetadata(contract.ControlPlane().MachineTemplate().Metadata().Get(clusterObjects.ControlPlane))
controlPlaneInfrastructureMachineTemplateTemplateMetadata := mustMetadata(contract.InfrastructureMachineTemplate().Template().Metadata().Get(clusterObjects.ControlPlaneInfrastructureMachineTemplate))

for _, machine := range clusterObjects.ControlPlaneMachines {
// ControlPlane Machine.metadata
g.Expect(machine.Labels).To(BeEquivalentTo(
machineMetadata := clusterv1.ObjectMeta{Labels: machine.Labels, Annotations: machine.Annotations}
if filterMetadataBeforeValidation != nil {
machineMetadata = filterMetadataBeforeValidation(machine)
}
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -457,9 +467,13 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster

// ControlPlane Machine InfrastructureMachine.metadata
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
infrastructureMachineMetadata := clusterv1.ObjectMeta{Labels: infrastructureMachine.GetLabels(), Annotations: infrastructureMachine.GetAnnotations()}
if filterMetadataBeforeValidation != nil {
infrastructureMachineMetadata = filterMetadataBeforeValidation(infrastructureMachine)
}
controlPlaneMachineTemplateInfrastructureRef, err := contract.ControlPlane().MachineTemplate().InfrastructureRef().Get(clusterObjects.ControlPlane)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -471,7 +485,7 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
controlPlaneInfrastructureMachineTemplateTemplateMetadata.Labels,
),
))
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(controlPlaneMachineTemplateInfrastructureRef),
Expand All @@ -484,7 +498,11 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster

// ControlPlane Machine BootstrapConfig.metadata
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
bootstrapConfigMetadata := clusterv1.ObjectMeta{Labels: bootstrapConfig.GetLabels(), Annotations: bootstrapConfig.GetAnnotations()}
if filterMetadataBeforeValidation != nil {
bootstrapConfigMetadata = filterMetadataBeforeValidation(bootstrapConfig)
}
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -497,16 +515,20 @@ func assertControlPlaneMachines(g Gomega, clusterObjects clusterObjects, cluster
))
g.Expect(
union(
bootstrapConfig.GetAnnotations(),
bootstrapConfigMetadata.Annotations,
).without(g, clusterv1.MachineCertificatesExpiryDateAnnotation),
).To(BeEquivalentTo(
controlPlaneMachineTemplateMetadata.Annotations,
))

// ControlPlane Machine Node.metadata
node := clusterObjects.NodesByMachine[machine.Name]
for k, v := range getManagedLabels(machine.Labels) {
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
nodeMetadata := clusterv1.ObjectMeta{Labels: node.GetLabels(), Annotations: node.GetAnnotations()}
if filterMetadataBeforeValidation != nil {
nodeMetadata = filterMetadataBeforeValidation(node)
}
for k, v := range getManagedLabels(machineMetadata.Labels) {
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
}
}
}
Expand Down Expand Up @@ -790,7 +812,7 @@ func assertMachineSets(g Gomega, clusterObjects clusterObjects, cluster *cluster
}
}

func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster) {
func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster *clusterv1.Cluster, filterMetadataBeforeValidation func(object client.Object) clusterv1.ObjectMeta) {
for _, machineDeployment := range clusterObjects.MachineDeployments {
mdTopology := getMDTopology(cluster, machineDeployment)
infrastructureMachineTemplate := clusterObjects.InfrastructureMachineTemplateByMachineDeployment[machineDeployment.Name]
Expand All @@ -802,8 +824,12 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
machineTemplateHash := machineSet.Labels[clusterv1.MachineDeploymentUniqueLabel]

for _, machine := range clusterObjects.MachinesByMachineSet[machineSet.Name] {
machineMetadata := clusterv1.ObjectMeta{Labels: machine.Labels, Annotations: machine.Annotations}
if filterMetadataBeforeValidation != nil {
machineMetadata = filterMetadataBeforeValidation(machine)
}
// MachineDeployment MachineSet Machine.metadata
g.Expect(machine.Labels).To(BeEquivalentTo(
g.Expect(machineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -816,13 +842,17 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
machineSet.Spec.Template.Labels,
),
))
g.Expect(machine.Annotations).To(BeEquivalentTo(
g.Expect(machineMetadata.Annotations).To(BeEquivalentTo(
machineSet.Spec.Template.Annotations,
))

// MachineDeployment MachineSet Machine InfrastructureMachine.metadata
infrastructureMachine := clusterObjects.InfrastructureMachineByMachine[machine.Name]
g.Expect(infrastructureMachine.GetLabels()).To(BeEquivalentTo(
infrastructureMachineMetadata := clusterv1.ObjectMeta{Labels: infrastructureMachine.GetLabels(), Annotations: infrastructureMachine.GetAnnotations()}
if filterMetadataBeforeValidation != nil {
infrastructureMachineMetadata = filterMetadataBeforeValidation(infrastructureMachine)
}
g.Expect(infrastructureMachineMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -836,7 +866,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
infrastructureMachineTemplateTemplateMetadata.Labels,
),
))
g.Expect(infrastructureMachine.GetAnnotations()).To(BeEquivalentTo(
g.Expect(infrastructureMachineMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(&machineSet.Spec.Template.Spec.InfrastructureRef),
Expand All @@ -849,7 +879,11 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster

// MachineDeployment MachineSet Machine BootstrapConfig.metadata
bootstrapConfig := clusterObjects.BootstrapConfigByMachine[machine.Name]
g.Expect(bootstrapConfig.GetLabels()).To(BeEquivalentTo(
bootstrapConfigMetadata := clusterv1.ObjectMeta{Labels: bootstrapConfig.GetLabels(), Annotations: bootstrapConfig.GetAnnotations()}
if filterMetadataBeforeValidation != nil {
bootstrapConfigMetadata = filterMetadataBeforeValidation(bootstrapConfig)
}
g.Expect(bootstrapConfigMetadata.Labels).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.ClusterNameLabel: cluster.Name,
Expand All @@ -863,7 +897,7 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster
bootstrapConfigTemplateTemplateMetadata.Labels,
),
))
g.Expect(bootstrapConfig.GetAnnotations()).To(BeEquivalentTo(
g.Expect(bootstrapConfigMetadata.Annotations).To(BeEquivalentTo(
union(
map[string]string{
clusterv1.TemplateClonedFromGroupKindAnnotation: groupKind(machineSet.Spec.Template.Spec.Bootstrap.ConfigRef),
Expand All @@ -876,8 +910,12 @@ func assertMachineSetsMachines(g Gomega, clusterObjects clusterObjects, cluster

// MachineDeployment MachineSet Machine Node.metadata
node := clusterObjects.NodesByMachine[machine.Name]
for k, v := range getManagedLabels(machine.Labels) {
g.Expect(node.GetLabels()).To(HaveKeyWithValue(k, v))
nodeMetadata := clusterv1.ObjectMeta{Labels: node.GetLabels(), Annotations: node.GetAnnotations()}
if filterMetadataBeforeValidation != nil {
nodeMetadata = filterMetadataBeforeValidation(node)
}
for k, v := range getManagedLabels(machineMetadata.Labels) {
g.Expect(nodeMetadata.Labels).To(HaveKeyWithValue(k, v))
}
}
}
Expand Down

0 comments on commit 6262c42

Please sign in to comment.