Skip to content

Commit

Permalink
Merge pull request #5311 from nawazkh/create_internal_lb
Browse files Browse the repository at this point in the history 
Use feature flag to create internal LB
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Dec 4, 2024
2 parents 983bf8b + 0ea69a2 commit 1275143
Show file tree
Hide file tree
Showing 4 changed files with 526 additions and 21 deletions.
4 changes: 2 additions & 2 deletions azure/scope/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure/services/subnets"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualnetworks"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/vnetpeerings"
"sigs.k8s.io/cluster-api-provider-azure/feature"
"sigs.k8s.io/cluster-api-provider-azure/util/futures"
"sigs.k8s.io/cluster-api-provider-azure/util/tele"
)
Expand Down Expand Up @@ -270,8 +271,7 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter {
},
}
}

if s.APIServerLB().Type != infrav1.Internal {
if s.APIServerLB().Type != infrav1.Internal && feature.Gates.Enabled(feature.APIServerILB) {
specs = append(specs, &loadbalancers.LBSpec{
Name: s.APIServerLB().Name + "-internal",
ResourceGroup: s.ResourceGroup(),
Expand Down
310 changes: 293 additions & 17 deletions azure/scope/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/utils/ptr"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
Expand All @@ -48,6 +50,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/subnets"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/vnetpeerings"
"sigs.k8s.io/cluster-api-provider-azure/feature"
)

const fakeClientID = "fake-client-id"
Expand Down Expand Up @@ -2174,6 +2177,7 @@ func TestBackendPoolName(t *testing.T) {
tests := []struct {
name string
clusterName string
featureGate featuregate.Feature

customAPIServerBackendPoolName string
customNodeBackendPoolName string
Expand All @@ -2190,6 +2194,14 @@ func TestBackendPoolName(t *testing.T) {
expectedNodeBackendPoolName: "NodeOutboundLBName-outboundBackendPool",
expectedControlPlaneBackendPoolName: "my-cluster-outbound-lb-outboundBackendPool",
},
{
name: "With default backend pool names feature gate enabled",
clusterName: "my-cluster",
featureGate: feature.APIServerILB,
expectedAPIServerBackendPoolName: "APIServerLBName-backendPool",
expectedNodeBackendPoolName: "NodeOutboundLBName-outboundBackendPool",
expectedControlPlaneBackendPoolName: "my-cluster-outbound-lb-outboundBackendPool",
},
{
name: "With custom node backend pool name",
clusterName: "my-cluster",
Expand Down Expand Up @@ -2218,6 +2230,9 @@ func TestBackendPoolName(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
g := NewWithT(t)
if tc.featureGate == feature.APIServerILB {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.Gates, tc.featureGate, true)()
}

cluster := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -2280,27 +2295,43 @@ func TestBackendPoolName(t *testing.T) {
}
clusterScope.AzureCluster.SetBackendPoolNameDefault()
got := clusterScope.LBSpecs()
g.Expect(got).To(HaveLen(4))
if tc.featureGate == feature.APIServerILB {
g.Expect(got).To(HaveLen(4))
} else {
g.Expect(got).To(HaveLen(3))
}

// API server backend pool name
apiServerLBSpec := got[0].(*loadbalancers.LBSpec)
g.Expect(apiServerLBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName))
g.Expect(apiServerLBSpec.Role).To(Equal(infrav1.APIServerRole))

// API server backend pool name
apiServerILBSpec := got[1].(*loadbalancers.LBSpec)
g.Expect(apiServerILBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName + "-internal"))
g.Expect(apiServerILBSpec.Role).To(Equal(infrav1.APIServerRoleInternal))

// Node backend pool name
NodeLBSpec := got[2].(*loadbalancers.LBSpec)
g.Expect(NodeLBSpec.BackendPoolName).To(Equal(tc.expectedNodeBackendPoolName))
g.Expect(NodeLBSpec.Role).To(Equal(infrav1.NodeOutboundRole))

// Control Plane backend pool name
controlPlaneLBSpec := got[3].(*loadbalancers.LBSpec)
g.Expect(controlPlaneLBSpec.BackendPoolName).To(Equal(tc.expectedControlPlaneBackendPoolName))
g.Expect(controlPlaneLBSpec.Role).To(Equal(infrav1.ControlPlaneOutboundRole))
if tc.featureGate == feature.APIServerILB {
// API server backend pool name
apiServerILBSpec := got[1].(*loadbalancers.LBSpec)
g.Expect(apiServerILBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName + "-internal"))
g.Expect(apiServerILBSpec.Role).To(Equal(infrav1.APIServerRoleInternal))

// Node backend pool name
NodeLBSpec := got[2].(*loadbalancers.LBSpec)
g.Expect(NodeLBSpec.BackendPoolName).To(Equal(tc.expectedNodeBackendPoolName))
g.Expect(NodeLBSpec.Role).To(Equal(infrav1.NodeOutboundRole))

// Control Plane backend pool name
controlPlaneLBSpec := got[3].(*loadbalancers.LBSpec)
g.Expect(controlPlaneLBSpec.BackendPoolName).To(Equal(tc.expectedControlPlaneBackendPoolName))
g.Expect(controlPlaneLBSpec.Role).To(Equal(infrav1.ControlPlaneOutboundRole))
} else {
// Node backend pool name
NodeLBSpec := got[1].(*loadbalancers.LBSpec)
g.Expect(NodeLBSpec.BackendPoolName).To(Equal(tc.expectedNodeBackendPoolName))
g.Expect(NodeLBSpec.Role).To(Equal(infrav1.NodeOutboundRole))

// Control Plane backend pool name
controlPlaneLBSpec := got[2].(*loadbalancers.LBSpec)
g.Expect(controlPlaneLBSpec.BackendPoolName).To(Equal(tc.expectedControlPlaneBackendPoolName))
g.Expect(controlPlaneLBSpec.Role).To(Equal(infrav1.ControlPlaneOutboundRole))
}
})
}
}
Expand Down Expand Up @@ -2611,6 +2642,7 @@ func TestFailureDomains(t *testing.T) {
func TestClusterScope_LBSpecs(t *testing.T) {
tests := []struct {
name string
featureGate featuregate.Feature
azureCluster *infrav1.AzureCluster
want []azure.ResourceSpecGetter
}{
Expand Down Expand Up @@ -2709,6 +2741,182 @@ func TestClusterScope_LBSpecs(t *testing.T) {
},
},
},
want: []azure.ResourceSpecGetter{
&loadbalancers.LBSpec{
Name: "api-server-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
SubnetName: "cp-subnet",
FrontendIPConfigs: []infrav1.FrontendIP{
{
Name: "api-server-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "api-server-lb-frontend-ip",
},
},
},
APIServerPort: 6443,
Type: infrav1.Public,
SKU: infrav1.SKUStandard,
Role: infrav1.APIServerRole,
BackendPoolName: "api-server-lb-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](30),
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
},
&loadbalancers.LBSpec{
Name: "node-outbound-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
FrontendIPConfigs: []infrav1.FrontendIP{
{
Name: "node-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "node-outbound-lb-frontend-ip",
},
},
},
Type: infrav1.Public,
SKU: infrav1.SKUStandard,
Role: infrav1.NodeOutboundRole,
BackendPoolName: "node-outbound-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](50),
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
},
&loadbalancers.LBSpec{
Name: "cp-outbound-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
FrontendIPConfigs: []infrav1.FrontendIP{
{
Name: "cp-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "cp-outbound-lb-frontend-ip",
},
},
},
Type: infrav1.Public,
SKU: infrav1.SKUStandard,
BackendPoolName: "cp-outbound-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](15),
Role: infrav1.ControlPlaneOutboundRole,
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
},
},
},
{
name: "API Server LB, Control Plane Oubound LB, and Node Outbound LB with feature gate",
featureGate: feature.APIServerILB,
azureCluster: &infrav1.AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-cluster",
},
Spec: infrav1.AzureClusterSpec{
AzureClusterClassSpec: infrav1.AzureClusterClassSpec{
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
SubscriptionID: "123",
Location: "westus2",
},
ControlPlaneEnabled: true,
ResourceGroup: "my-rg",
NetworkSpec: infrav1.NetworkSpec{
Vnet: infrav1.VnetSpec{
Name: "my-vnet",
ResourceGroup: "my-rg",
},
Subnets: []infrav1.SubnetSpec{
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "cp-subnet",
Role: infrav1.SubnetControlPlane,
},
},
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "node-subnet",
Role: infrav1.SubnetNode,
},
},
},
APIServerLB: &infrav1.LoadBalancerSpec{
Name: "api-server-lb",
BackendPool: infrav1.BackendPool{
Name: "api-server-lb-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Public,
IdleTimeoutInMinutes: ptr.To[int32](30),
SKU: infrav1.SKUStandard,
},
FrontendIPs: []infrav1.FrontendIP{
{
Name: "api-server-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "api-server-lb-frontend-ip",
},
},
},
},
ControlPlaneOutboundLB: &infrav1.LoadBalancerSpec{
Name: "cp-outbound-lb",
BackendPool: infrav1.BackendPool{
Name: "cp-outbound-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Public,
IdleTimeoutInMinutes: ptr.To[int32](15),
SKU: infrav1.SKUStandard,
},
FrontendIPs: []infrav1.FrontendIP{
{
Name: "cp-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "cp-outbound-lb-frontend-ip",
},
},
},
},
NodeOutboundLB: &infrav1.LoadBalancerSpec{
Name: "node-outbound-lb",
BackendPool: infrav1.BackendPool{
Name: "node-outbound-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Public,
IdleTimeoutInMinutes: ptr.To[int32](50),
SKU: infrav1.SKUStandard,
},
FrontendIPs: []infrav1.FrontendIP{
{
Name: "node-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "node-outbound-lb-frontend-ip",
},
},
},
},
},
},
},
want: []azure.ResourceSpecGetter{
&loadbalancers.LBSpec{
Name: "api-server-lb",
Expand Down Expand Up @@ -2882,11 +3090,79 @@ func TestClusterScope_LBSpecs(t *testing.T) {
},
},
},
{
name: "Private API Server LB",
featureGate: feature.APIServerILB,
azureCluster: &infrav1.AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-cluster",
},
Spec: infrav1.AzureClusterSpec{
AzureClusterClassSpec: infrav1.AzureClusterClassSpec{
SubscriptionID: "123",
Location: "westus2",
},
ControlPlaneEnabled: true,
ResourceGroup: "my-rg",
NetworkSpec: infrav1.NetworkSpec{
Vnet: infrav1.VnetSpec{
Name: "my-vnet",
ResourceGroup: "my-rg",
},
Subnets: []infrav1.SubnetSpec{
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "cp-subnet",
Role: infrav1.SubnetControlPlane,
},
},
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "node-subnet",
Role: infrav1.SubnetNode,
},
},
},
APIServerLB: &infrav1.LoadBalancerSpec{
Name: "api-server-lb",
BackendPool: infrav1.BackendPool{
Name: "api-server-lb-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Internal,
IdleTimeoutInMinutes: ptr.To[int32](30),
SKU: infrav1.SKUStandard,
},
},
},
},
},
want: []azure.ResourceSpecGetter{
&loadbalancers.LBSpec{
Name: "api-server-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
SubnetName: "cp-subnet",
APIServerPort: 6443,
Type: infrav1.Internal,
SKU: infrav1.SKUStandard,
Role: infrav1.APIServerRole,
BackendPoolName: "api-server-lb-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](30),
AdditionalTags: infrav1.Tags{},
},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

if tc.featureGate == feature.APIServerILB {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.Gates, tc.featureGate, true)()
}
cluster := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: tc.azureCluster.Name,
Expand Down
Loading

0 comments on commit 1275143

Please sign in to comment.