openstack: add unit tests for waitServerComplete{Building,Setup}

Signed-off-by: Zeyad Gouda <[email protected]>

spread/export_openstack_test.go

@@ -1,6 +1,8 @@
 package spread
 
 import (
+	"time"
+
 	"github.com/go-goose/goose/v5/glance"
 )
 
@@ -9,8 +11,9 @@ var (
 )
 
 type (
-	OpenstackProvider = openstackProvider
-	GlanceImageClient = glanceImageClient
+	OpenstackProvider   = openstackProvider
+	GlanceImageClient   = glanceImageClient
+	OpenstackServerData = openstackServerData
 )
 
 func MockOpenstackImageClient(opst *OpenstackProvider, newIC glanceImageClient) (restore func()) {
@@ -21,10 +24,56 @@ func MockOpenstackImageClient(opst *OpenstackProvider, newIC glanceImageClient)
 	}
 }
 
+func MockOpenstackComputeClient(opst *OpenstackProvider, newIC novaComputeClient) (restore func()) {
+	oldNovaImageClient := opst.computeClient
+	opst.computeClient = newIC
+	return func() {
+		opst.computeClient = oldNovaImageClient
+	}
+}
+
+func MockOpenstackBuildingTimeout(timeout, retry time.Duration) (restore func()) {
+	oldTimeout := openstackBuildingTimeout
+	oldRetry := openstackBuildingRetry
+	openstackBuildingTimeout = timeout
+	openstackBuildingRetry = retry
+	return func() {
+		openstackBuildingTimeout = oldTimeout
+		openstackBuildingRetry = oldRetry
+	}
+}
+
+func MockOpenstackSetupTimeout(timeout, retry time.Duration) (restore func()) {
+	oldTimeout := openstackSetupTimeout
+	oldRetry := openstackSetupRetry
+	openstackSetupTimeout = timeout
+	openstackSetupRetry = retry
+	return func() {
+		openstackSetupTimeout = oldTimeout
+		openstackSetupRetry = oldRetry
+	}
+}
+
 func (opst *OpenstackProvider) FindImage(name string) (*glance.ImageDetail, error) {
 	return opst.findImage(name)
 }
 
+func (opst *openstackProvider) WaitServerCompleteBuilding(serverData OpenstackServerData) error {
+	server := &openstackServer{
+		p: opst,
+		d: serverData,
+	}
+	return opst.waitServerCompleteBuilding(server)
+}
+
+func (opst *openstackProvider) WaitServerCompleteSetup(serverData OpenstackServerData) error {
+	server := &openstackServer{
+		p: opst,
+		d: serverData,
+	}
+	return opst.waitServerCompleteSetup(server)
+}
+
 func NewOpenstackError(gooseError error) error {
 	return &openstackError{gooseError}
 }

spread/openstack.go

@@ -30,13 +30,22 @@ type glanceImageClient interface {
 	ListImagesDetail() ([]glance.ImageDetail, error)
 }
 
+type novaComputeClient interface {
+	ListFlavors() ([]nova.Entity, error)
+	ListAvailabilityZones() ([]nova.AvailabilityZone, error)
+	GetServer(serverId string) (*nova.ServerDetail, error)
+	ListServersDetail(filter *nova.Filter) ([]nova.ServerDetail, error)
+	RunServer(opts nova.RunServerOpts) (*nova.Entity, error)
+	DeleteServer(serverId string) error
+}
+
 type openstackProvider struct {
 	project *Project
 	backend *Backend
 	options *Options
 
 	region        string
-	computeClient *nova.Client
+	computeClient novaComputeClient
 	networkClient *neutron.Client
 	imageClient   glanceImageClient
 
@@ -341,9 +350,12 @@ func (p *openstackProvider) findSecurityGroupNames(names []string) ([]nova.Secur
 	return secGroupNames, nil
 }
 
+var openstackBuildingTimeout = 2 * time.Minute
+var openstackBuildingRetry = 5 * time.Second
+
 func (p *openstackProvider) waitServerCompleteBuilding(s *openstackServer) error {
-	timeout := time.After(2 * time.Minute)
-	retry := time.NewTicker(5 * time.Second)
+	timeout := time.After(openstackBuildingTimeout)
+	retry := time.NewTicker(openstackBuildingRetry)
 	defer retry.Stop()
 
 	// Wait until the server is actually running
@@ -369,6 +381,9 @@ func (p *openstackProvider) waitServerCompleteBuilding(s *openstackServer) error
 	}
 }
 
+var openstackSetupTimeout = 5 * time.Minute
+var openstackSetupRetry = 5 * time.Second
+
 func (p *openstackProvider) waitServerCompleteSetup(s *openstackServer) error {
 	server, err := p.computeClient.GetServer(s.d.Id)
 	if err != nil {
@@ -396,16 +411,16 @@ func (p *openstackProvider) waitServerCompleteSetup(s *openstackServer) error {
 
 	// Iterate until the ssh connection to the host can be stablished
 	// In openstack the client cannot access to the serial console of the instance
-	timeout := time.After(5 * time.Minute)
-	retry := time.NewTicker(5 * time.Second)
+	timeout := time.After(openstackSetupTimeout)
+	retry := time.NewTicker(openstackSetupRetry)
 	defer retry.Stop()
 
 	for {
 		select {
 		case <-timeout:
-			return &FatalError{fmt.Errorf("cannot ssh to the allocated instance")}
+			return &FatalError{fmt.Errorf("cannot ssh to the allocated instance: timeout reached")}
 		case <-retry.C:
-			_, err = ssh.Dial("tcp", addr, config)
+			_, err = sshDial("tcp", addr, config)
 			if err == nil {
 				debugf("Connection to server %s established", s.d.Name)
 				return nil

spread/openstack_test.go

@@ -8,6 +8,8 @@ import (
 	"time"
 
 	"github.com/go-goose/goose/v5/glance"
+	"github.com/go-goose/goose/v5/nova"
+	"golang.org/x/crypto/ssh"
 
 	"github.com/snapcore/spread/spread"
 
@@ -82,10 +84,44 @@ func (ic *fakeGlanceImageClient) ListImagesDetail() ([]glance.ImageDetail, error
 	return ic.res, ic.err
 }
 
+type fakeNovaComputeClient struct {
+	listFlavors           func() ([]nova.Entity, error)
+	listAvailabilityZones func() ([]nova.AvailabilityZone, error)
+	getServer             func(serverId string) (*nova.ServerDetail, error)
+	listServersDetail     func(filter *nova.Filter) ([]nova.ServerDetail, error)
+	runServer             func(opts nova.RunServerOpts) (*nova.Entity, error)
+	deleteServer          func(serverId string) error
+}
+
+func (cc *fakeNovaComputeClient) ListFlavors() ([]nova.Entity, error) {
+	return cc.listFlavors()
+}
+
+func (cc *fakeNovaComputeClient) ListAvailabilityZones() ([]nova.AvailabilityZone, error) {
+	return cc.listAvailabilityZones()
+}
+
+func (cc *fakeNovaComputeClient) GetServer(serverId string) (*nova.ServerDetail, error) {
+	return cc.getServer(serverId)
+}
+
+func (cc *fakeNovaComputeClient) ListServersDetail(filter *nova.Filter) ([]nova.ServerDetail, error) {
+	return cc.listServersDetail(filter)
+}
+
+func (cc *fakeNovaComputeClient) RunServer(opts nova.RunServerOpts) (*nova.Entity, error) {
+	return cc.runServer(opts)
+}
+
+func (cc *fakeNovaComputeClient) DeleteServer(serverId string) error {
+	return cc.deleteServer(serverId)
+}
+
 type openstackFindImageSuite struct {
 	opst *spread.OpenstackProvider
 
-	fakeImageClient *fakeGlanceImageClient
+	fakeImageClient   *fakeGlanceImageClient
+	fakeComputeClient *fakeNovaComputeClient
 }
 
 var _ = Suite(&openstackFindImageSuite{})
@@ -94,8 +130,10 @@ func (s *openstackFindImageSuite) SetUpTest(c *C) {
 	s.opst = newOpenstack()
 	c.Assert(s.opst, NotNil)
 	s.fakeImageClient = &fakeGlanceImageClient{}
+	s.fakeComputeClient = &fakeNovaComputeClient{}
 
 	spread.MockOpenstackImageClient(s.opst, s.fakeImageClient)
+	spread.MockOpenstackComputeClient(s.opst, s.fakeComputeClient)
 }
 
 func (s *openstackFindImageSuite) TestOpenstackFindImageNotFound(c *C) {
@@ -206,3 +244,180 @@ func (s *openstackFindImageSuite) TestOpenstackFindImageComplex(c *C) {
 		}
 	}
 }
+
+func (s *openstackFindImageSuite) TestOpenstackWaitServerCompleteBuildingHappy(c *C) {
+	serverData := spread.OpenstackServerData{
+		Id: "test-id",
+	}
+
+	count := 0
+	s.fakeComputeClient.getServer = func(serverId string) (*nova.ServerDetail, error) {
+		count++
+		c.Check(serverId, Equals, serverData.Id)
+		switch count {
+		case 1:
+			return nil, errors.New("bad id")
+		case 2:
+			server := nova.ServerDetail{
+				Id:     serverId,
+				Status: nova.StatusActive,
+			}
+			return &server, nil
+		}
+		c.Fatalf("should not reach here")
+		return nil, nil
+	}
+
+	restore := spread.MockOpenstackBuildingTimeout(100*time.Millisecond, 1*time.Nanosecond)
+	defer restore()
+
+	err := s.opst.WaitServerCompleteBuilding(serverData)
+	c.Check(err, IsNil)
+	c.Check(count, Equals, 2)
+}
+
+func (s *openstackFindImageSuite) TestOpenstackWaitServerCompleteBuildingBadStatus(c *C) {
+	serverData := spread.OpenstackServerData{
+		Id: "test-id",
+	}
+
+	count := 0
+	s.fakeComputeClient.getServer = func(serverId string) (*nova.ServerDetail, error) {
+		count++
+		c.Check(serverId, Equals, serverData.Id)
+		switch count {
+		case 1:
+			return nil, errors.New("bad id")
+		case 2:
+			server := nova.ServerDetail{
+				Id:     serverId,
+				Status: nova.StatusError,
+			}
+			return &server, nil
+		}
+		return nil, nil
+	}
+
+	restore := spread.MockOpenstackBuildingTimeout(100*time.Millisecond, 1*time.Nanosecond)
+	defer restore()
+
+	err := s.opst.WaitServerCompleteBuilding(serverData)
+	c.Check(err, ErrorMatches, "cannot use server: status is not active but ERROR")
+	c.Check(count, Equals, 2)
+}
+
+func (s *openstackFindImageSuite) TestOpenstackWaitServerCompleteBuildingTimeout(c *C) {
+	s.fakeComputeClient.getServer = func(serverId string) (*nova.ServerDetail, error) {
+		return nil, nil
+	}
+
+	restore := spread.MockOpenstackBuildingTimeout(1*time.Nanosecond, 1*time.Hour)
+	defer restore()
+
+	err := s.opst.WaitServerCompleteBuilding(spread.OpenstackServerData{})
+	c.Check(err, ErrorMatches, "cannot check status: timeout reached")
+}
+
+func (s *openstackFindImageSuite) TestOpenstackWaitServerCompleteSetupHappy(c *C) {
+	serverData := spread.OpenstackServerData{
+		Id:       "test-id",
+		Networks: []string{"net-1"},
+	}
+
+	count := 0
+	spread.MockSshDial(func(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) {
+		count++
+		switch count {
+		case 1:
+			return nil, errors.New("connection error")
+		case 2:
+			return &ssh.Client{}, nil
+		}
+		c.Fatalf("should not reach here")
+		return nil, nil
+	})
+
+	s.fakeComputeClient.getServer = func(serverId string) (*nova.ServerDetail, error) {
+		c.Check(serverId, Equals, serverData.Id)
+		server := nova.ServerDetail{
+			Id:     serverId,
+			Status: nova.StatusActive,
+			Addresses: map[string][]nova.IPAddress{
+				"net-1": {{Address: "10.0.0.1"}},
+			},
+		}
+		return &server, nil
+	}
+
+	restore := spread.MockOpenstackSetupTimeout(100*time.Millisecond, 1*time.Nanosecond)
+	defer restore()
+
+	err := s.opst.WaitServerCompleteSetup(serverData)
+	c.Check(err, IsNil)
+	c.Check(count, Equals, 2)
+}
+
+func (s *openstackFindImageSuite) TestOpenstackWaitServerCompleteSetupTimeout(c *C) {
+	serverData := spread.OpenstackServerData{
+		Id:       "test-id",
+		Networks: []string{"net-1"},
+	}
+
+	spread.MockSshDial(func(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) {
+		return nil, errors.New("connection error")
+	})
+	s.fakeComputeClient.getServer = func(serverId string) (*nova.ServerDetail, error) {
+		c.Check(serverId, Equals, serverData.Id)
+		server := nova.ServerDetail{
+			Id:     serverId,
+			Status: nova.StatusActive,
+			Addresses: map[string][]nova.IPAddress{
+				"net-1": {{Address: "10.0.0.1"}},
+			},
+		}
+		return &server, nil
+	}
+
+	restore := spread.MockOpenstackSetupTimeout(1*time.Nanosecond, 1*time.Hour)
+	defer restore()
+
+	err := s.opst.WaitServerCompleteSetup(serverData)
+	c.Check(err, ErrorMatches, "cannot ssh to the allocated instance: timeout reached")
+}
+
+func (s *openstackFindImageSuite) TestOpenstackWaitServerCompleteSetupBadAddress(c *C) {
+	serverData := spread.OpenstackServerData{
+		Id:       "test-id",
+		Networks: []string{"net-1"},
+	}
+
+	spread.MockSshDial(func(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) {
+		return nil, errors.New("connection error")
+	})
+	s.fakeComputeClient.getServer = func(serverId string) (*nova.ServerDetail, error) {
+		c.Check(serverId, Equals, serverData.Id)
+		server := nova.ServerDetail{
+			Id:        serverId,
+			Status:    nova.StatusActive,
+			Addresses: map[string][]nova.IPAddress{},
+		}
+		return &server, nil
+	}
+
+	err := s.opst.WaitServerCompleteSetup(serverData)
+	c.Check(err, ErrorMatches, "cannot retrieve server address")
+}
+
+func (s *openstackFindImageSuite) TestOpenstackWaitServerCompleteSetupBadServerId(c *C) {
+	serverData := spread.OpenstackServerData{
+		Id: "test-id",
+	}
+
+	s.fakeComputeClient.getServer = func(serverId string) (*nova.ServerDetail, error) {
+		c.Check(serverId, Equals, serverData.Id)
+		return nil, errors.New("cannot get server")
+	}
+
+	err := s.opst.WaitServerCompleteSetup(serverData)
+	c.Check(err, ErrorMatches, "cannot retrieving server information: cannot get server")
+}