From 69a5f209570f1862c55b202dbca96b13c86b091c Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Tue, 22 Nov 2022 22:51:57 +0530 Subject: [PATCH 1/2] pvt cluster changes --- api/v1beta1/azurecluster_validation.go | 4 +++- azure/services/loadbalancers/loadbalancers.go | 19 +++++++++++++++++-- azure/services/loadbalancers/spec.go | 18 +++++++++++++++--- config/default/manager_image_patch.yaml | 2 +- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/api/v1beta1/azurecluster_validation.go b/api/v1beta1/azurecluster_validation.go index 6ccc737e097..de4bf20c306 100644 --- a/api/v1beta1/azurecluster_validation.go +++ b/api/v1beta1/azurecluster_validation.go @@ -19,6 +19,7 @@ package v1beta1 import ( "fmt" "net" + "os" "reflect" "regexp" @@ -339,7 +340,8 @@ func validateAPIServerLB(lb LoadBalancerSpec, old LoadBalancerSpec, cidrs []stri fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP")); err != nil { allErrs = append(allErrs, err) } - if len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { + lbIPAllocationMethod := os.Getenv("AZURE_APISERVER_LB_IP_ALLOCATION") + if lbIPAllocationMethod == "Static" && len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { allErrs = append(allErrs, field.Forbidden(fldPath.Child("name"), "API Server load balancer private IP should not be modified after AzureCluster creation.")) } } diff --git a/azure/services/loadbalancers/loadbalancers.go b/azure/services/loadbalancers/loadbalancers.go index 49aa430ee78..e8120a76ecb 100644 --- a/azure/services/loadbalancers/loadbalancers.go +++ b/azure/services/loadbalancers/loadbalancers.go @@ -18,6 +18,9 @@ package loadbalancers import ( "context" + "fmt" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network" + "github.com/pkg/errors" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" @@ -78,13 +81,25 @@ func (s *Service) Reconcile(ctx context.Context) error { // Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (i.e. error creating) -> operationNotDoneError (i.e. creating in progress) -> no error (i.e. created) var result error for _, lbSpec := range specs { - if _, err := s.CreateResource(ctx, lbSpec, serviceName); err != nil { + if lb, err := s.CreateResource(ctx, lbSpec, serviceName); err != nil { if !azure.IsOperationNotDoneError(err) || result == nil { result = err } + } else { + loadBalancer, ok := lb.(network.LoadBalancer) + if !ok { + // Return out of loop since this would be an unexepcted fatal error + result = errors.Errorf("created resource %T is not a network.loadBalancer", result) + break + } + if lbSpec.ResourceName() == s.Scope.APIServerLB().Name { + if len(*loadBalancer.FrontendIPConfigurations) > 0 && *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress != "" { + fmt.Println("LOADBALANCER's PRIVATE IP", (*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress) + s.Scope.APIServerLB().FrontendIPs[0].PrivateIPAddress = *(*loadBalancer.FrontendIPConfigurations)[0].PrivateIPAddress + } + } } } - s.Scope.UpdatePutStatus(infrav1.LoadBalancersReadyCondition, serviceName, result) return result } diff --git a/azure/services/loadbalancers/spec.go b/azure/services/loadbalancers/spec.go index 50c96ccc7b8..a18c4995f9d 100644 --- a/azure/services/loadbalancers/spec.go +++ b/azure/services/loadbalancers/spec.go @@ -20,6 +20,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" + "os" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" @@ -161,15 +162,26 @@ func (s *LBSpec) Parameters(existing interface{}) (parameters interface{}, err e func getFrontendIPConfigs(lbSpec LBSpec) ([]network.FrontendIPConfiguration, []network.SubResource) { frontendIPConfigurations := make([]network.FrontendIPConfiguration, 0) frontendIDs := make([]network.SubResource, 0) + lbIPAllocationMethod := os.Getenv("AZURE_APISERVER_LB_IP_ALLOCATION") + var privateIPAllocationMethod network.IPAllocationMethod + if lbIPAllocationMethod == "Dynamic" { + privateIPAllocationMethod = network.IPAllocationMethodDynamic + } else { + privateIPAllocationMethod = network.IPAllocationMethodStatic + } for _, ipConfig := range lbSpec.FrontendIPConfigs { var properties network.FrontendIPConfigurationPropertiesFormat + var privateIPAddress string + if lbIPAllocationMethod == "Static" { + privateIPAddress = ipConfig.PrivateIPAddress + } if lbSpec.Type == infrav1.Internal { properties = network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.IPAllocationMethodStatic, + PrivateIPAllocationMethod: privateIPAllocationMethod, Subnet: &network.Subnet{ ID: to.StringPtr(azure.SubnetID(lbSpec.SubscriptionID, lbSpec.VNetResourceGroup, lbSpec.VNetName, lbSpec.SubnetName)), }, - PrivateIPAddress: to.StringPtr(ipConfig.PrivateIPAddress), + PrivateIPAddress: to.StringPtr(privateIPAddress), } } else { properties = network.FrontendIPConfigurationPropertiesFormat{ @@ -225,7 +237,7 @@ func getLoadBalancingRules(lbSpec LBSpec, frontendIDs []network.SubResource) []n FrontendPort: to.Int32Ptr(lbSpec.APIServerPort), BackendPort: to.Int32Ptr(lbSpec.APIServerPort), IdleTimeoutInMinutes: lbSpec.IdleTimeoutInMinutes, - EnableFloatingIP: to.BoolPtr(false), + EnableFloatingIP: to.BoolPtr(true), LoadDistribution: network.LoadDistributionDefault, FrontendIPConfiguration: &frontendIPConfig, BackendAddressPool: &network.SubResource{ diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index 0876a1db40d..68c9e417def 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + - image: gcr.io/spectro-common-dev/cluster-api-azure-controller:dev name: manager From da059d833fef79416121a373c5a1c4ee3408026c Mon Sep 17 00:00:00 2001 From: Akhilesh Verma Date: Wed, 23 Nov 2022 14:25:08 +0530 Subject: [PATCH 2/2] more changes --- api/v1beta1/azurecluster_default.go | 4 ++++ api/v1beta1/azurecluster_validation.go | 4 +--- api/v1beta1/types_class.go | 2 ++ azure/scope/cluster.go | 1 + azure/services/loadbalancers/spec.go | 13 +++---------- 5 files changed, 11 insertions(+), 13 deletions(-) diff --git a/api/v1beta1/azurecluster_default.go b/api/v1beta1/azurecluster_default.go index ccd4d13eb6a..ea646dd2fc1 100644 --- a/api/v1beta1/azurecluster_default.go +++ b/api/v1beta1/azurecluster_default.go @@ -18,6 +18,7 @@ package v1beta1 import ( "fmt" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network" "k8s.io/utils/pointer" ) @@ -307,6 +308,9 @@ func (lb *LoadBalancerClassSpec) setAPIServerLBDefaults() { if lb.SKU == "" { lb.SKU = SKUStandard } + if lb.IPAllocationMethod == "" { + lb.IPAllocationMethod = string(network.IPAllocationMethodDynamic) + } if lb.IdleTimeoutInMinutes == nil { lb.IdleTimeoutInMinutes = pointer.Int32Ptr(DefaultOutboundRuleIdleTimeoutInMinutes) } diff --git a/api/v1beta1/azurecluster_validation.go b/api/v1beta1/azurecluster_validation.go index de4bf20c306..388fc483569 100644 --- a/api/v1beta1/azurecluster_validation.go +++ b/api/v1beta1/azurecluster_validation.go @@ -19,7 +19,6 @@ package v1beta1 import ( "fmt" "net" - "os" "reflect" "regexp" @@ -340,8 +339,7 @@ func validateAPIServerLB(lb LoadBalancerSpec, old LoadBalancerSpec, cidrs []stri fldPath.Child("frontendIPConfigs").Index(0).Child("privateIP")); err != nil { allErrs = append(allErrs, err) } - lbIPAllocationMethod := os.Getenv("AZURE_APISERVER_LB_IP_ALLOCATION") - if lbIPAllocationMethod == "Static" && len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { + if lb.IPAllocationMethod == "Static" && len(old.FrontendIPs) != 0 && old.FrontendIPs[0].PrivateIPAddress != lb.FrontendIPs[0].PrivateIPAddress { allErrs = append(allErrs, field.Forbidden(fldPath.Child("name"), "API Server load balancer private IP should not be modified after AzureCluster creation.")) } } diff --git a/api/v1beta1/types_class.go b/api/v1beta1/types_class.go index e2b1aceddfe..33fc805ba27 100644 --- a/api/v1beta1/types_class.go +++ b/api/v1beta1/types_class.go @@ -87,6 +87,8 @@ type LoadBalancerClassSpec struct { SKU SKU `json:"sku,omitempty"` // +optional Type LBType `json:"type,omitempty"` + // +optional + IPAllocationMethod string `json:"ipAllocationMethod,omitempty"` // IdleTimeoutInMinutes specifies the timeout for the TCP idle connection. // +optional IdleTimeoutInMinutes *int32 `json:"idleTimeoutInMinutes,omitempty"` diff --git a/azure/scope/cluster.go b/azure/scope/cluster.go index 22eb23e1213..dd922eb3eec 100644 --- a/azure/scope/cluster.go +++ b/azure/scope/cluster.go @@ -244,6 +244,7 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter { Type: s.APIServerLB().Type, SKU: infrav1.SKUStandard, Role: infrav1.APIServerRole, + IPAllocationMethod: s.APIServerLB().IPAllocationMethod, BackendPoolName: s.APIServerLBPoolName(s.APIServerLB().Name), IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes, AdditionalTags: s.AdditionalTags(), diff --git a/azure/services/loadbalancers/spec.go b/azure/services/loadbalancers/spec.go index a18c4995f9d..84406779ff6 100644 --- a/azure/services/loadbalancers/spec.go +++ b/azure/services/loadbalancers/spec.go @@ -20,7 +20,6 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" - "os" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" @@ -42,6 +41,7 @@ type LBSpec struct { BackendPoolName string FrontendIPConfigs []infrav1.FrontendIP APIServerPort int32 + IPAllocationMethod string IdleTimeoutInMinutes *int32 AdditionalTags map[string]string } @@ -162,22 +162,15 @@ func (s *LBSpec) Parameters(existing interface{}) (parameters interface{}, err e func getFrontendIPConfigs(lbSpec LBSpec) ([]network.FrontendIPConfiguration, []network.SubResource) { frontendIPConfigurations := make([]network.FrontendIPConfiguration, 0) frontendIDs := make([]network.SubResource, 0) - lbIPAllocationMethod := os.Getenv("AZURE_APISERVER_LB_IP_ALLOCATION") - var privateIPAllocationMethod network.IPAllocationMethod - if lbIPAllocationMethod == "Dynamic" { - privateIPAllocationMethod = network.IPAllocationMethodDynamic - } else { - privateIPAllocationMethod = network.IPAllocationMethodStatic - } for _, ipConfig := range lbSpec.FrontendIPConfigs { var properties network.FrontendIPConfigurationPropertiesFormat var privateIPAddress string - if lbIPAllocationMethod == "Static" { + if lbSpec.IPAllocationMethod == "Static" { privateIPAddress = ipConfig.PrivateIPAddress } if lbSpec.Type == infrav1.Internal { properties = network.FrontendIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: privateIPAllocationMethod, + PrivateIPAllocationMethod: network.IPAllocationMethod(lbSpec.IPAllocationMethod), Subnet: &network.Subnet{ ID: to.StringPtr(azure.SubnetID(lbSpec.SubscriptionID, lbSpec.VNetResourceGroup, lbSpec.VNetName, lbSpec.SubnetName)), },