F #783 Fix randomly failing GOCA tests (#6789)

* Fix virtual network not ready tests * Adds exponential backoff helper and corrects hook test * Fixes GOCA context and VM test Signed-off-by: Jaime <[email protected]> Signed-off-by: Aleix Ramírez <[email protected]> (cherry picked from commit f091804)
OpenNebula · Nov 20, 2024 · 9d5da2d · 9d5da2d
1 parent bb029bb
commit 9d5da2d
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 35 deletions.
diff --git a/src/oca/go/src/goca/context_test.go b/src/oca/go/src/goca/context_test.go
@@ -37,7 +37,7 @@ func (s *ContextSuite) TestDefaultContext(c *C) {
 
 func (s *ContextSuite) TestContextTimeout(c *C) {
 	// Simple example, which shows the timeout is triggered
-	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
+	ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
 	defer cancel()
 
 	userC := testCtrl.User(0)

diff --git a/src/oca/go/src/goca/helper_test.go b/src/oca/go/src/goca/helper_test.go
@@ -19,6 +19,7 @@ package goca
 import (
 	"crypto/md5"
 	"fmt"
+	"math"
 	"strconv"
 	"testing"
 	"time"
@@ -51,15 +52,45 @@ func WaitResource(f func() bool) bool {
 func GetUserGroup(t *testing.T, user string) (string, error) {
 	uid, err := testCtrl.Users().ByName(user)
 	if err != nil {
-        t.Error("Cannot retreive caller user ID")
+		t.Error("Cannot retreive caller user ID")
 	}
 
-    // Get User Info
-    u, err := testCtrl.User(uid).Info(false)
+	// Get User Info
+	u, err := testCtrl.User(uid).Info(false)
 	if err != nil {
-        t.Error("Cannot retreive caller user Info")
+		t.Error("Cannot retreive caller user Info")
 	}
 
-    return u.GName, nil
+	return u.GName, nil
 
 }
+
+// Retries function with exponential backoff until maxRetries are reached
+// * fn: function to retry until returns nil as error
+// * delayMs: base delay time in milliseconds
+// * maxRetries: maximum number of retries
+// * initDelayMs: The initial delay time in milliseconds (before the first function call)
+// * maxDelayMs: The maximum number of milliseconds between retries
+func retryWithExponentialBackoff(fn func() error, delayMs int, maxRetries int, initDelayMs int, maxDelayMs int) error {
+
+	start := time.Now()
+	time.Sleep(time.Duration(initDelayMs) * time.Millisecond)
+
+	delay := float64(delayMs)
+	maxDelay := float64(maxDelayMs)
+
+	for retries := 0; retries <= maxRetries; retries++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+		time.Sleep(time.Duration(delay) * time.Millisecond)
+		delay *= math.Pow(2, float64(retries))
+		if delay > maxDelay {
+			delay = maxDelay
+		}
+	}
+	totalTime := time.Since(start)
+
+	return fmt.Errorf("retry limit reached (%d) after (%f) seconds", maxRetries, totalTime.Seconds())
+}
diff --git a/src/oca/go/src/goca/hook_test.go b/src/oca/go/src/goca/hook_test.go
@@ -17,8 +17,8 @@
 package goca
 
 import (
+	"fmt"
 	"testing"
-	"time"
 
 	hk "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/hook"
 	"github.com/OpenNebula/one/src/oca/go/src/goca/schemas/hook/keys"
@@ -84,29 +84,31 @@ func TestHook(t *testing.T) {
 	//triger the hook
 	testCtrl.Zones().ServerRaftStatus()
 
-	time.Sleep(2 * time.Second)
-
-	hook, err = hookC.Info(false)
+	checkLogExecution := func() error {
+		hook, err = hookC.Info(false)
+		if len(hook.Log.ExecutionRecords) <= currentExecs {
+			return fmt.Errorf("Hook have not been triggered")
+		}
+		return nil
+	}
 
-	if len(hook.Log.ExecutionRecords) <= currentExecs {
-		t.Errorf("Hook have not been triggered")
+	err = retryWithExponentialBackoff(checkLogExecution, 1000, 5, 2000, 3000)
+	if err != nil {
+		t.Errorf("Hook have not been triggered: %s", err)
 	}
 
 	// Check retry functionality
 	currentExecs = len(hook.Log.ExecutionRecords)
 
 	hookC.Retry(hook.Log.ExecutionRecords[0].ExecId)
 
-	time.Sleep(2 * time.Second)
-
-	hook, err = hookC.Info(false)
-
-	if len(hook.Log.ExecutionRecords) <= currentExecs {
-		t.Errorf("Hook execution have not been retried")
+	err = retryWithExponentialBackoff(checkLogExecution, 1000, 5, 2000, 3000)
+	if err != nil {
+		t.Errorf("Hook execution has not been retried: %s", err)
 	}
 
 	if hook.Log.ExecutionRecords[len(hook.Log.ExecutionRecords)-1].Retry != "yes" {
-		t.Errorf("Hook execution have not been retried")
+		t.Errorf("Hook execution has not been retried")
 	}
 
 	// Delete template

diff --git a/src/oca/go/src/goca/virtualnetwork_test.go b/src/oca/go/src/goca/virtualnetwork_test.go
@@ -82,6 +82,9 @@ func TestVirtualNetwork(t *testing.T) {
 	}
 
 	vnetC := testCtrl.VirtualNetwork(id)
+
+	WaitState(t, vnetC, "READY")
+
 	vnet, err = vnetC.Info(false)
 	if err != nil {
 		t.Error(err)
@@ -162,10 +165,10 @@ func TestVirtualNetworkRecover(t *testing.T) {
 	var err error
 
 	vnTpl := "NAME = vn_invalid_ar\n" +
-			 "BRIDGE = vbr0\n" +
-			 "VN_MAD = dummy\n" +
-			 "NETWORK_ADDRESS = 192.168.0.0\n"+
-			 "AR = [ TYPE = IP4, IP = 192.168.0.1, SIZE = -1 ]\n"
+		"BRIDGE = vbr0\n" +
+		"VN_MAD = dummy\n" +
+		"NETWORK_ADDRESS = 192.168.0.0\n" +
+		"AR = [ TYPE = IP4, IP = 192.168.0.1, SIZE = -1 ]\n"
 
 	id, err := testCtrl.VirtualNetworks().Create(vnTpl, -1)
 	if err != nil {

diff --git a/src/oca/go/src/goca/virtualrouter_test.go b/src/oca/go/src/goca/virtualrouter_test.go
@@ -185,6 +185,8 @@ func TestVirtualRouter(t *testing.T) {
 	vn_tmpl.Add(vnkeys.VNMad, "dummy")
 
 	vnet_id, _ := testCtrl.VirtualNetworks().Create(vn_tmpl.String(), 0)
+	vnetC := testCtrl.VirtualNetwork(vnet_id)
+	WaitState(t, vnetC, "READY")
 
 	nic_tmpl := shared.NewNIC()
 	nic_tmpl.Add(shared.Network, "go-net")

diff --git a/src/oca/go/src/goca/vm_test.go b/src/oca/go/src/goca/vm_test.go
@@ -1,3 +1,4 @@
+//go:build !disabled
 // +build !disabled
 
 /* -------------------------------------------------------------------------- */
@@ -19,9 +20,10 @@
 package goca
 
 import (
-	"testing"
-	"strconv"
+	"fmt"
 	"regexp"
+	"strconv"
+	"testing"
 
 	ds "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/datastore"
 	dskeys "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/datastore/keys"
@@ -67,22 +69,22 @@ func (s *VMSuite) SetUpSuite(c *C) {
 	testCtrl.Datastore(0).Update(tmpl.String(), 1)
 
 	dsTmpl := "NAME = go_backup_ds\n" +
-			  "DS_MAD = dummy\n" +
-			  "TM_MAD = dummy\n" +
-			  "TYPE = BACKUP_DS\n";
+		"DS_MAD = dummy\n" +
+		"TM_MAD = dummy\n" +
+		"TYPE = BACKUP_DS\n"
 
 	s.dsID, _ = testCtrl.Datastores().Create(dsTmpl, 0)
 
 	vnTpl := "NAME = vn_go_test_sg\n" +
-			 "BRIDGE = vbr0\n" +
-			 "VN_MAD = dummy\n" +
-			 "NETWORK_ADDRESS = 192.168.0.0\n"+
-			 "AR = [ TYPE = IP4, IP = 192.168.0.1, SIZE = 254 ]\n"
+		"BRIDGE = vbr0\n" +
+		"VN_MAD = dummy\n" +
+		"NETWORK_ADDRESS = 192.168.0.0\n" +
+		"AR = [ TYPE = IP4, IP = 192.168.0.1, SIZE = 254 ]\n"
 	s.vnID, _ = testCtrl.VirtualNetworks().Create(vnTpl, -1)
 
 	sgTpl := "NAME = sg_go_nic_attach\n" +
-			 "DESCRIPTION  = \"test security group\"\n" +
-			 "ATT1 = \"VAL1\"\n"
+		"DESCRIPTION  = \"test security group\"\n" +
+		"ATT1 = \"VAL1\"\n"
 	s.sgID, _ = testCtrl.SecurityGroups().Create(sgTpl)
 }
 
@@ -106,10 +108,21 @@ func (s *VMSuite) TearDownSuite(c *C) {
 	c.Assert(err, IsNil)
 
 	for i := 0; i < len(backupDS.Images.ID); i++ {
-	err := testCtrl.Image(backupDS.Images.ID[i]).Delete()
+		err := testCtrl.Image(backupDS.Images.ID[i]).Delete()
 		c.Assert(err, IsNil)
 	}
 
+	//wait for images to be deleted
+	err = retryWithExponentialBackoff(func() error {
+		backupDS, err := testCtrl.Datastore(s.dsID).Info(false)
+		c.Assert(err, IsNil)
+		if len(backupDS.Images.ID) > 0 {
+			return fmt.Errorf("still exist images")
+		}
+		return nil
+	}, 1000, 5, 0, 3000)
+	c.Assert(err, IsNil)
+
 	err = testCtrl.Template(s.templateID).Delete()
 	c.Assert(err, IsNil)