test: Stabilize for FVTs (#429)

- Increase the initial time to stabilize for predictor tests - Scale to zero before AllowAnyPVC tests --------- Signed-off-by: Christian Kadner <[email protected]>
kserve · Sep 13, 2023 · d4fc59a · d4fc59a
1 parent 5dfc278
commit d4fc59a
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 5 deletions.
diff --git a/fvt/fvtclient.go b/fvt/fvtclient.go
@@ -59,8 +59,8 @@ import (
 	torchserveapi "github.com/kserve/modelmesh-serving/fvt/generated/torchserve/apis"
 )
 
-const PredictorTimeout = time.Second * 120       // absolute time to wait for predictor to become ready
-const TimeForStatusToStabilize = time.Second * 5 // time to wait between watcher events before assuming a stable state
+const PredictorTimeout = time.Second * 120        // absolute time to wait for predictor to become ready
+const TimeForStatusToStabilize = time.Second * 10 // time to wait between watcher events before assuming a stable state
 
 type ModelServingConnectionType int
 

diff --git a/fvt/predictor/predictor_suite_test.go b/fvt/predictor/predictor_suite_test.go
@@ -56,7 +56,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	FVTClientInstance.CreateTLSSecrets()
 
 	// ensure a stable deploy state
-	WaitForStableActiveDeployState(time.Second * 30)
+	WaitForStableActiveDeployState(time.Second * 45)
 
 	return nil
 }, func(_ []byte) {

diff --git a/fvt/storage/storage_test.go b/fvt/storage/storage_test.go
@@ -130,11 +130,21 @@ var _ = Describe("ISVCs", func() {
 			//   from the old to the new pod
 
 			// make a shallow copy of default configmap (don't modify the DefaultConfig reference)
-			// keeping 1 pod per runtime and don't scale to 0
 			config := make(map[string]interface{})
 			for k, v := range DefaultConfig {
 				config[k] = v
 			}
+
+			// scale to 0 for resource-constrained environments (only 2 CPUs on GH actions)
+			// to stop and remove runtimes which are not used for this test
+			//   Warning   FailedScheduling   pod/modelmesh-serving-mlserver-1.x-...
+			//     0/1 nodes are available: 1 Insufficient cpu. preemption: 0/1 nodes are available:
+			//       1 No preemption victims found for incoming pod.
+			config["scaleToZero"] = map[string]interface{}{
+				"enabled":            true,
+				"gracePeriodSeconds": 5,
+			}
+
 			// update the model-serving-config to allow any PVC
 			config["allowAnyPVC"] = true
 
@@ -194,11 +204,21 @@ var _ = Describe("ISVCs", func() {
 
 		It("should fail with non-existent PVC", func() {
 			// make a shallow copy of default configmap (don't modify the DefaultConfig reference)
-			// keeping 1 pod per runtime and don't scale to 0
 			config := make(map[string]interface{})
 			for k, v := range DefaultConfig {
 				config[k] = v
 			}
+
+			// scale to 0 for resource-constrained environments (only 2 CPUs on GH actions)
+			// to stop and remove runtimes which are not used for this test
+			//   Warning   FailedScheduling   pod/modelmesh-serving-mlserver-1.x-...
+			//     0/1 nodes are available: 1 Insufficient cpu. preemption: 0/1 nodes are available:
+			//       1 No preemption victims found for incoming pod.
+			config["scaleToZero"] = map[string]interface{}{
+				"enabled":            true,
+				"gracePeriodSeconds": 5,
+			}
+
 			// update the model-serving-config to allow any PVC
 			config["allowAnyPVC"] = true
 			FVTClientInstance.ApplyUserConfigMap(config)