From 999fb71c76436ba8c5dd618f5fdb9698cd448b2c Mon Sep 17 00:00:00 2001 From: Shai Oren Date: Thu, 25 Jan 2024 13:02:46 +0200 Subject: [PATCH] Fixed stability of powersave test --- .../powermanagementhelper.go | 16 +++++++++++++--- .../powermanagementparams.go | 2 +- tests/ranfunc/powermanagement/tests/powersave.go | 15 +++++++++------ .../openshift-kni/eco-goinfra/pkg/mco/mcp.go | 2 ++ 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/ranfunc/powermanagement/internal/powermanagementhelper/powermanagementhelper.go b/tests/ranfunc/powermanagement/internal/powermanagementhelper/powermanagementhelper.go index 89c8f2a6..40a670dc 100644 --- a/tests/ranfunc/powermanagement/internal/powermanagementhelper/powermanagementhelper.go +++ b/tests/ranfunc/powermanagement/internal/powermanagementhelper/powermanagementhelper.go @@ -23,6 +23,7 @@ import ( "github.com/openshift-kni/eco-goinfra/pkg/pod" "github.com/openshift-kni/eco-gosystem/tests/internal/cmd" "github.com/openshift-kni/eco-gosystem/tests/internal/config" + mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/openshift-kni/eco-gosystem/tests/internal/inittools" "github.com/openshift-kni/eco-gosystem/tests/ranfunc/internal/ranfuncinittools" @@ -185,8 +186,10 @@ func RedefineContainerResources( return pod } -// SetPowerMode updates the performance profile with the given workload hints, and waits for the mcp update. -func SetPowerMode(perfProfile *nto.Builder, perPodPowerManagement, highPowerConsumption, realTime bool) error { +// SetPowerModeAndWaitForMcpUpdate updates the performance profile with the given workload hints, +// and waits for the mcp update. +func SetPowerModeAndWaitForMcpUpdate(perfProfile *nto.Builder, node nodes.Builder, perPodPowerManagement, + highPowerConsumption, realTime bool) error { glog.V(100).Infof("Set powersave mode on performance profile") perfProfile.Definition.Spec.WorkloadHints = &performancev2.WorkloadHints{ @@ -203,7 +206,14 @@ func SetPowerMode(perfProfile *nto.Builder, perPodPowerManagement, highPowerCons mcp, err := mco.Pull(ranfuncinittools.HubAPIClient, "master") Expect(err).ToNot(HaveOccurred()) - err = mcp.WaitForUpdate(powermanagementparams.Timeout) + err = mcp.WaitToBeInCondition(mcov1.MachineConfigPoolUpdating, corev1.ConditionTrue, 15*time.Minute) + Expect(err).ToNot(HaveOccurred()) + + err = mcp.WaitToBeInCondition(mcov1.MachineConfigPoolUpdated, corev1.ConditionTrue, 25*time.Minute) + Expect(err).ToNot(HaveOccurred()) + + err = node.WaitUntilReady(5 * time.Minute) + Expect(err).ToNot(HaveOccurred()) return err } diff --git a/tests/ranfunc/powermanagement/internal/powermanagementparams/powermanagementparams.go b/tests/ranfunc/powermanagement/internal/powermanagementparams/powermanagementparams.go index d1150dc0..19b95d96 100644 --- a/tests/ranfunc/powermanagement/internal/powermanagementparams/powermanagementparams.go +++ b/tests/ranfunc/powermanagement/internal/powermanagementparams/powermanagementparams.go @@ -52,7 +52,7 @@ const ( // ProcessExporterImage is the image of the process-exporter pod. ProcessExporterImage = "quay.io/ocp-edge-qe/process-exporter:ppid-2" // Timeout is the timeout being used in powersave tests. - Timeout = 15 * time.Minute + Timeout = 30 * time.Minute // CnfTestImage is the test image used by pods. CnfTestImage = "quay.io/openshift-kni/cnf-tests:4.8" // PrivPodNamespace is the priv pod namespace. diff --git a/tests/ranfunc/powermanagement/tests/powersave.go b/tests/ranfunc/powermanagement/tests/powersave.go index c1a1bcd4..54a4068b 100644 --- a/tests/ranfunc/powermanagement/tests/powersave.go +++ b/tests/ranfunc/powermanagement/tests/powersave.go @@ -19,6 +19,7 @@ import ( "github.com/openshift-kni/eco-gosystem/tests/ranfunc/powermanagement/internal/powermanagementparams" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" + mcov1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/kubernetes/pkg/kubelet/cm/cpuset" @@ -35,8 +36,6 @@ var _ = Describe("Per-Core Runtime Tuning of power states - CRI-O", Ordered, fun originPerformanceProfileSpec performancev2.PerformanceProfileSpec ) - var timeout = 15 * time.Minute - BeforeAll(func() { // Get nodes for connection host nodeList, err = nodes.List(ranfuncinittools.HubAPIClient) @@ -49,7 +48,7 @@ var _ = Describe("Per-Core Runtime Tuning of power states - CRI-O", Ordered, fun Expect(err).ToNot(HaveOccurred()) snoNode = nodeList[0].Object - originPerformanceProfileSpec = perfProfile.Object.Spec + // originPerformanceProfileSpec = perfProfile.Object.Spec Expect(err).ToNot(HaveOccurred()) }) @@ -63,8 +62,12 @@ var _ = Describe("Per-Core Runtime Tuning of power states - CRI-O", Ordered, fun mcp, err := mco.Pull(ranfuncinittools.HubAPIClient, "master") Expect(err).ToNot(HaveOccurred()) - err = mcp.WaitForUpdate(timeout) + err = mcp.WaitToBeInCondition(mcov1.MachineConfigPoolUpdating, corev1.ConditionTrue, 15*time.Minute) + Expect(err).ToNot(HaveOccurred()) + + err = mcp.WaitToBeInCondition(mcov1.MachineConfigPoolUpdated, corev1.ConditionTrue, 25*time.Minute) Expect(err).ToNot(HaveOccurred()) + }) // OCP-54571 - Install SNO node with standard DU profile that does not include WorkloadHints @@ -104,7 +107,7 @@ var _ = Describe("Per-Core Runtime Tuning of power states - CRI-O", Ordered, fun // OCP-54572 - Enable powersave at node level and then enable performance at node level It("Enable powersave at node level and then enable performance at node level", func() { By("Patching the performance profile with the workload hints") - err := powermanagementhelper.SetPowerMode(perfProfile, true, false, true) + err := powermanagementhelper.SetPowerModeAndWaitForMcpUpdate(perfProfile, *nodeList[0], true, false, true) Expect(err).ToNot(HaveOccurred(), "Unable to set power mode") cmdline, err := cmd.ExecCmd([]string{"chroot", "rootfs", "cat", "/proc/cmdline"}, snoNode.Name) @@ -130,7 +133,7 @@ var _ = Describe("Per-Core Runtime Tuning of power states - CRI-O", Ordered, fun memLimit := resource.MustParse("100Mi") By("Patching the performance profile with the workload hints") - err := powermanagementhelper.SetPowerMode(perfProfile, true, false, true) + err := powermanagementhelper.SetPowerModeAndWaitForMcpUpdate(perfProfile, *nodeList[0], true, false, true) Expect(err).ToNot(HaveOccurred(), "Unable to set power mode") By("Define test pod") diff --git a/vendor/github.com/openshift-kni/eco-goinfra/pkg/mco/mcp.go b/vendor/github.com/openshift-kni/eco-goinfra/pkg/mco/mcp.go index 273c564c..4df34a33 100644 --- a/vendor/github.com/openshift-kni/eco-goinfra/pkg/mco/mcp.go +++ b/vendor/github.com/openshift-kni/eco-goinfra/pkg/mco/mcp.go @@ -218,6 +218,7 @@ func (builder *MCPBuilder) WaitForUpdate(timeout time.Duration) error { } for _, condition := range mcpUpdating.Status.Conditions { + if condition.Type == "Updating" && condition.Status == isTrue { err := wait.PollUntilContextTimeout( context.TODO(), fiveScds, timeout, true, func(ctx context.Context) (bool, error) { @@ -230,6 +231,7 @@ func (builder *MCPBuilder) WaitForUpdate(timeout time.Duration) error { for _, condition := range mcpUpdated.Status.Conditions { if condition.Type == "Updated" && condition.Status == isTrue { + return true, nil } }