Skip to content

Commit

Permalink
Check all resources realized status when creating subnet
Browse files Browse the repository at this point in the history 
Subnet includes child resources which also need checking realized status.
Check subnet for all child resources
  • Loading branch information
@TaoZou1
TaoZou1 committed Nov 22, 2024
1 parent a2702f2 commit 6e4d6ed
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 6 deletions.
43 changes: 41 additions & 2 deletions pkg/nsx/services/realizestate/realize_state.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,29 @@
package realizestate

import (
"errors"
"fmt"

"github.com/vmware/vsphere-automation-sdk-go/services/nsxt/model"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"

"github.com/vmware-tanzu/nsx-operator/pkg/logger"
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
)

var (
log = &logger.Log
)

const (
RealizedLogicalSwitch = "RealizedLogicalSwitch"
RealizedLogicalPort = "RealizedLogicalPort"
RealizedLogicalRouterPort = "RealizedLogicalRouterPort"
RealizedLogicalRouter = "RealizedLogicalRouter"
)

type RealizeStateService struct {
common.Service
}
Expand Down Expand Up @@ -43,6 +56,8 @@ func IsRealizeStateError(err error) bool {

// CheckRealizeState allows the caller to check realize status of entityType with retries.
// backoff defines the maximum retries and the wait interval between two retries.
// if entityType == "", check all the entities, all entities should be in the REALIZED state to be tereated as REALIZED
// if there are multiple entities with the same entityType, one REALIZED state will be treated as REALIZED
func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, intentPath, entityType string) error {
// TODO， ask NSX if there were multiple realize states could we check only the latest one?
return retry.OnError(backoff, func(err error) bool {
Expand All @@ -54,12 +69,19 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
if err != nil {
return err
}
entitiesStatus := make(map[string]bool)
entitiesMsg := make(map[string]string)
for _, result := range results.Results {
if entityType != "" && result.EntityType != nil && *result.EntityType != entityType {
continue
}
entity, exists := entitiesStatus[*result.EntityType]
if !exists {
entitiesStatus[*result.EntityType] = false
}
if *result.State == model.GenericPolicyRealizedResource_STATE_REALIZED {
return nil
entitiesStatus[*result.EntityType] = true
continue
}
if *result.State == model.GenericPolicyRealizedResource_STATE_ERROR {
var errMsg []string
Expand All @@ -68,9 +90,26 @@ func (service *RealizeStateService) CheckRealizeState(backoff wait.Backoff, inte
errMsg = append(errMsg, *alarm.Message)
}
}
return NewRealizeStateError(fmt.Sprintf("%s realized with errors: %s", entityType, errMsg))
if !(exists && entity) {
if entityType != "" && entityType == *result.EntityType || entityType == "" {
log.Error(errors.New("realize state error"), "entityType", *result.EntityType, "path", intentPath)
}
entitiesMsg[*result.EntityType] = fmt.Sprintf("%s realized with errors: %s", *result.EntityType, errMsg)
}
}
}
result := true
for key, value := range entitiesStatus {
result = result && value
if !value {
if err, exists := entitiesMsg[key]; exists {
return NewRealizeStateError(err)
}
}
}
if result {
return nil
}
return fmt.Errorf("%s not realized", entityType)
})
}
122 changes: 121 additions & 1 deletion pkg/nsx/services/realizestate/realize_state_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ func TestRealizeStateService_CheckRealizeState(t *testing.T) {
},
}, nil
})
defer patches.Reset()

backoff := wait.Backoff{
Duration: 1 * time.Second,
Expand All @@ -76,4 +75,125 @@ func TestRealizeStateService_CheckRealizeState(t *testing.T) {
realizeStateError, ok = err.(*RealizeStateError)
assert.True(t, ok)
assert.Equal(t, realizeStateError.Error(), "RealizedLogicalPort realized with errors: [mocked error]")

// multil entries for the same resource
patches.Reset()
patches = gomonkey.ApplyFunc((*fakeRealizedEntitiesClient).List, func(c *fakeRealizedEntitiesClient, intentPathParam string, sitePathParam *string) (model.GenericPolicyRealizedResourceListResult, error) {
return model.GenericPolicyRealizedResourceListResult{
Results: []model.GenericPolicyRealizedResource{
{
State: common.String(model.GenericPolicyRealizedResource_STATE_ERROR),
Alarms: []model.PolicyAlarmResource{
{Message: common.String("mocked error")},
},
EntityType: common.String("RealizedLogicalPort"),
},
{
State: common.String(model.GenericPolicyRealizedResource_STATE_REALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalPort"),
},
},
}, nil
})
err = s.CheckRealizeState(backoff, "/orgs/default/projects/project-quality/vpcs/vpc/subnets/subnet/ports/port", "RealizedLogicalPort")
assert.Equal(t, err, nil)

// for subnet, RealizedLogicalPort realized with errors
patches.Reset()

patches = gomonkey.ApplyFunc((*fakeRealizedEntitiesClient).List, func(c *fakeRealizedEntitiesClient, intentPathParam string, sitePathParam *string) (model.GenericPolicyRealizedResourceListResult, error) {
return model.GenericPolicyRealizedResourceListResult{
Results: []model.GenericPolicyRealizedResource{
{
State: common.String(model.GenericPolicyRealizedResource_STATE_ERROR),
Alarms: []model.PolicyAlarmResource{
{Message: common.String("mocked error")},
},
EntityType: common.String("RealizedLogicalPort"),
},
{
State: common.String(model.GenericPolicyRealizedResource_STATE_UNREALIZED),
Alarms: []model.PolicyAlarmResource{
{Message: common.String("mocked error")},
},
EntityType: common.String("RealizedLogicalSwitch"),
},
{
State: common.String(model.GenericPolicyRealizedResource_STATE_REALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalRouterPort"),
},
},
}, nil
})
err = s.CheckRealizeState(backoff, "/orgs/default/projects/project-quality/vpcs/vpc/subnets/subnet/", "")

realizeStateError, ok = err.(*RealizeStateError)
assert.True(t, ok)
assert.Equal(t, realizeStateError.Error(), "RealizedLogicalPort realized with errors: [mocked error]")

// for subnet, realized successfully
patches.Reset()

patches = gomonkey.ApplyFunc((*fakeRealizedEntitiesClient).List, func(c *fakeRealizedEntitiesClient, intentPathParam string, sitePathParam *string) (model.GenericPolicyRealizedResourceListResult, error) {
return model.GenericPolicyRealizedResourceListResult{
Results: []model.GenericPolicyRealizedResource{
{
State: common.String(model.GenericPolicyRealizedResource_STATE_REALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalPort"),
},
{
State: common.String(model.GenericPolicyRealizedResource_STATE_REALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalSwitch"),
},
{
State: common.String(model.GenericPolicyRealizedResource_STATE_REALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalRouterPort"),
},
},
}, nil
})
err = s.CheckRealizeState(backoff, "/orgs/default/projects/project-quality/vpcs/vpc/subnets/subnet/", "")
assert.Equal(t, err, nil)

// for subnet, need retry
patches.Reset()

patches = gomonkey.ApplyFunc((*fakeRealizedEntitiesClient).List, func(c *fakeRealizedEntitiesClient, intentPathParam string, sitePathParam *string) (model.GenericPolicyRealizedResourceListResult, error) {
return model.GenericPolicyRealizedResourceListResult{
Results: []model.GenericPolicyRealizedResource{
{
State: common.String(model.GenericPolicyRealizedResource_STATE_REALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalPort"),
},
{
State: common.String(model.GenericPolicyRealizedResource_STATE_UNREALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalSwitch"),
},
{
State: common.String(model.GenericPolicyRealizedResource_STATE_REALIZED),
Alarms: []model.PolicyAlarmResource{},
EntityType: common.String("RealizedLogicalRouterPort"),
},
},
}, nil
})
backoff = wait.Backoff{
Duration: 10 * time.Millisecond,
Factor: 1,
Jitter: 0,
Steps: 1,
}
err = s.CheckRealizeState(backoff, "/orgs/default/projects/project-quality/vpcs/vpc/subnets/subnet/", "")
assert.NotEqual(t, err, nil)
_, ok = err.(*RealizeStateError)
assert.Equal(t, ok, false)
patches.Reset()

}
6 changes: 4 additions & 2 deletions pkg/nsx/services/subnet/subnet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,9 @@ func (service *SubnetService) createOrUpdateSubnet(obj client.Object, nsxSubnet
// Failure of CheckRealizeState may result in the creation of an existing Subnet.
// For Subnets, it's important to reuse the already created NSXSubnet.
// For SubnetSets, since the ID includes a random value, the created NSX Subnet needs to be deleted and recreated.
if err = realizeService.CheckRealizeState(backoff, *nsxSubnet.Path, "RealizedLogicalSwitch"); err != nil {
log.Error(err, "Failed to check subnet realization state", "ID", *nsxSubnet.Id)

if err = realizeService.CheckRealizeState(backoff, *nsxSubnet.Path, ""); err != nil {
log.Error(err, "Failed to check subnet realization state", "ID", *nsxSubnet.Id, "resource", "RealizedSubnet")
// Delete the subnet if realization check fails, avoiding creating duplicate subnets continuously.
deleteErr := service.DeleteSubnet(*nsxSubnet)
if deleteErr != nil {
Expand All @@ -165,6 +166,7 @@ func (service *SubnetService) createOrUpdateSubnet(obj client.Object, nsxSubnet
}
return "", err
}

if err = service.SubnetStore.Apply(nsxSubnet); err != nil {
log.Error(err, "Failed to add subnet to store", "ID", *nsxSubnet.Id)
return "", err
Expand Down
2 changes: 1 addition & 1 deletion pkg/nsx/services/vpc/vpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -854,7 +854,7 @@ func (s *VPCService) createNSXVPC(createdVpc *model.Vpc, nc *common.VPCNetworkCo
func (s *VPCService) checkVPCRealizationState(createdVpc *model.Vpc, newVpcPath string) error {
log.V(2).Info("Check VPC realization state", "VPC", *createdVpc.Id)
realizeService := realizestate.InitializeRealizeState(s.Service)
if err := realizeService.CheckRealizeState(util.NSXTDefaultRetry, newVpcPath, "RealizedLogicalRouter"); err != nil {
if err := realizeService.CheckRealizeState(util.NSXTDefaultRetry, newVpcPath, realizestate.RealizedLogicalRouter); err != nil {
log.Error(err, "Failed to check VPC realization state", "VPC", *createdVpc.Id)
if realizestate.IsRealizeStateError(err) {
log.Error(err, "The created VPC is in error realization state, cleaning the resource", "VPC", *createdVpc.Id)
Expand Down

0 comments on commit 6e4d6ed

Please sign in to comment.