Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use feature flag to create internal LB #5311

Merged
merged 1 commit into from
Dec 4, 2024
Merged

Use feature flag to create internal LB #5311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions azure/scope/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure/services/subnets"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/virtualnetworks"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/vnetpeerings"
"sigs.k8s.io/cluster-api-provider-azure/feature"
"sigs.k8s.io/cluster-api-provider-azure/util/futures"
"sigs.k8s.io/cluster-api-provider-azure/util/tele"
)
Expand Down Expand Up @@ -270,8 +271,7 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter {
},
}
}

if s.APIServerLB().Type != infrav1.Internal {
if s.APIServerLB().Type != infrav1.Internal && feature.Gates.Enabled(feature.APIServerILB) {
specs = append(specs, &loadbalancers.LBSpec{
Name: s.APIServerLB().Name + "-internal",
ResourceGroup: s.ResourceGroup(),
Expand Down
310 changes: 293 additions & 17 deletions azure/scope/cluster_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/component-base/featuregate"
featuregatetesting "k8s.io/component-base/featuregate/testing"
"k8s.io/utils/ptr"
clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
Expand All @@ -48,6 +50,7 @@ import (
"sigs.k8s.io/cluster-api-provider-azure/azure/services/securitygroups"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/subnets"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/vnetpeerings"
"sigs.k8s.io/cluster-api-provider-azure/feature"
)

const fakeClientID = "fake-client-id"
Expand Down Expand Up @@ -2174,6 +2177,7 @@ func TestBackendPoolName(t *testing.T) {
tests := []struct {
name string
clusterName string
featureGate featuregate.Feature

customAPIServerBackendPoolName string
customNodeBackendPoolName string
Expand All @@ -2190,6 +2194,14 @@ func TestBackendPoolName(t *testing.T) {
expectedNodeBackendPoolName: "NodeOutboundLBName-outboundBackendPool",
expectedControlPlaneBackendPoolName: "my-cluster-outbound-lb-outboundBackendPool",
},
{
name: "With default backend pool names feature gate enabled",
clusterName: "my-cluster",
featureGate: feature.APIServerILB,
expectedAPIServerBackendPoolName: "APIServerLBName-backendPool",
expectedNodeBackendPoolName: "NodeOutboundLBName-outboundBackendPool",
expectedControlPlaneBackendPoolName: "my-cluster-outbound-lb-outboundBackendPool",
},
{
name: "With custom node backend pool name",
clusterName: "my-cluster",
Expand Down Expand Up @@ -2218,6 +2230,9 @@ func TestBackendPoolName(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
g := NewWithT(t)
if tc.featureGate == feature.APIServerILB {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.Gates, tc.featureGate, true)()
}

cluster := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -2280,27 +2295,43 @@ func TestBackendPoolName(t *testing.T) {
}
clusterScope.AzureCluster.SetBackendPoolNameDefault()
got := clusterScope.LBSpecs()
g.Expect(got).To(HaveLen(4))
if tc.featureGate == feature.APIServerILB {
g.Expect(got).To(HaveLen(4))
} else {
g.Expect(got).To(HaveLen(3))
}

// API server backend pool name
apiServerLBSpec := got[0].(*loadbalancers.LBSpec)
g.Expect(apiServerLBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName))
g.Expect(apiServerLBSpec.Role).To(Equal(infrav1.APIServerRole))

// API server backend pool name
apiServerILBSpec := got[1].(*loadbalancers.LBSpec)
g.Expect(apiServerILBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName + "-internal"))
g.Expect(apiServerILBSpec.Role).To(Equal(infrav1.APIServerRoleInternal))

// Node backend pool name
NodeLBSpec := got[2].(*loadbalancers.LBSpec)
g.Expect(NodeLBSpec.BackendPoolName).To(Equal(tc.expectedNodeBackendPoolName))
g.Expect(NodeLBSpec.Role).To(Equal(infrav1.NodeOutboundRole))

// Control Plane backend pool name
controlPlaneLBSpec := got[3].(*loadbalancers.LBSpec)
g.Expect(controlPlaneLBSpec.BackendPoolName).To(Equal(tc.expectedControlPlaneBackendPoolName))
g.Expect(controlPlaneLBSpec.Role).To(Equal(infrav1.ControlPlaneOutboundRole))
if tc.featureGate == feature.APIServerILB {
// API server backend pool name
apiServerILBSpec := got[1].(*loadbalancers.LBSpec)
g.Expect(apiServerILBSpec.BackendPoolName).To(Equal(tc.expectedAPIServerBackendPoolName + "-internal"))
g.Expect(apiServerILBSpec.Role).To(Equal(infrav1.APIServerRoleInternal))

// Node backend pool name
NodeLBSpec := got[2].(*loadbalancers.LBSpec)
g.Expect(NodeLBSpec.BackendPoolName).To(Equal(tc.expectedNodeBackendPoolName))
g.Expect(NodeLBSpec.Role).To(Equal(infrav1.NodeOutboundRole))

// Control Plane backend pool name
controlPlaneLBSpec := got[3].(*loadbalancers.LBSpec)
g.Expect(controlPlaneLBSpec.BackendPoolName).To(Equal(tc.expectedControlPlaneBackendPoolName))
g.Expect(controlPlaneLBSpec.Role).To(Equal(infrav1.ControlPlaneOutboundRole))
} else {
// Node backend pool name
NodeLBSpec := got[1].(*loadbalancers.LBSpec)
g.Expect(NodeLBSpec.BackendPoolName).To(Equal(tc.expectedNodeBackendPoolName))
g.Expect(NodeLBSpec.Role).To(Equal(infrav1.NodeOutboundRole))

// Control Plane backend pool name
controlPlaneLBSpec := got[2].(*loadbalancers.LBSpec)
g.Expect(controlPlaneLBSpec.BackendPoolName).To(Equal(tc.expectedControlPlaneBackendPoolName))
g.Expect(controlPlaneLBSpec.Role).To(Equal(infrav1.ControlPlaneOutboundRole))
}
})
}
}
Expand Down Expand Up @@ -2611,6 +2642,7 @@ func TestFailureDomains(t *testing.T) {
func TestClusterScope_LBSpecs(t *testing.T) {
tests := []struct {
name string
featureGate featuregate.Feature
azureCluster *infrav1.AzureCluster
want []azure.ResourceSpecGetter
}{
Expand Down Expand Up @@ -2709,6 +2741,182 @@ func TestClusterScope_LBSpecs(t *testing.T) {
},
},
},
want: []azure.ResourceSpecGetter{
&loadbalancers.LBSpec{
Name: "api-server-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
SubnetName: "cp-subnet",
FrontendIPConfigs: []infrav1.FrontendIP{
{
Name: "api-server-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "api-server-lb-frontend-ip",
},
},
},
APIServerPort: 6443,
Type: infrav1.Public,
SKU: infrav1.SKUStandard,
Role: infrav1.APIServerRole,
BackendPoolName: "api-server-lb-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](30),
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
},
&loadbalancers.LBSpec{
Name: "node-outbound-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
FrontendIPConfigs: []infrav1.FrontendIP{
{
Name: "node-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "node-outbound-lb-frontend-ip",
},
},
},
Type: infrav1.Public,
SKU: infrav1.SKUStandard,
Role: infrav1.NodeOutboundRole,
BackendPoolName: "node-outbound-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](50),
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
},
&loadbalancers.LBSpec{
Name: "cp-outbound-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
FrontendIPConfigs: []infrav1.FrontendIP{
{
Name: "cp-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "cp-outbound-lb-frontend-ip",
},
},
},
Type: infrav1.Public,
SKU: infrav1.SKUStandard,
BackendPoolName: "cp-outbound-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](15),
Role: infrav1.ControlPlaneOutboundRole,
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
},
},
},
{
name: "API Server LB, Control Plane Oubound LB, and Node Outbound LB with feature gate",
featureGate: feature.APIServerILB,
azureCluster: &infrav1.AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-cluster",
},
Spec: infrav1.AzureClusterSpec{
AzureClusterClassSpec: infrav1.AzureClusterClassSpec{
AdditionalTags: infrav1.Tags{
"foo": "bar",
},
SubscriptionID: "123",
Location: "westus2",
},
ControlPlaneEnabled: true,
ResourceGroup: "my-rg",
NetworkSpec: infrav1.NetworkSpec{
Vnet: infrav1.VnetSpec{
Name: "my-vnet",
ResourceGroup: "my-rg",
},
Subnets: []infrav1.SubnetSpec{
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "cp-subnet",
Role: infrav1.SubnetControlPlane,
},
},
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "node-subnet",
Role: infrav1.SubnetNode,
},
},
},
APIServerLB: &infrav1.LoadBalancerSpec{
Name: "api-server-lb",
BackendPool: infrav1.BackendPool{
Name: "api-server-lb-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Public,
IdleTimeoutInMinutes: ptr.To[int32](30),
SKU: infrav1.SKUStandard,
},
FrontendIPs: []infrav1.FrontendIP{
{
Name: "api-server-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "api-server-lb-frontend-ip",
},
},
},
},
ControlPlaneOutboundLB: &infrav1.LoadBalancerSpec{
Name: "cp-outbound-lb",
BackendPool: infrav1.BackendPool{
Name: "cp-outbound-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Public,
IdleTimeoutInMinutes: ptr.To[int32](15),
SKU: infrav1.SKUStandard,
},
FrontendIPs: []infrav1.FrontendIP{
{
Name: "cp-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "cp-outbound-lb-frontend-ip",
},
},
},
},
NodeOutboundLB: &infrav1.LoadBalancerSpec{
Name: "node-outbound-lb",
BackendPool: infrav1.BackendPool{
Name: "node-outbound-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Public,
IdleTimeoutInMinutes: ptr.To[int32](50),
SKU: infrav1.SKUStandard,
},
FrontendIPs: []infrav1.FrontendIP{
{
Name: "node-outbound-lb-frontend-ip",
PublicIP: &infrav1.PublicIPSpec{
Name: "node-outbound-lb-frontend-ip",
},
},
},
},
},
},
},
want: []azure.ResourceSpecGetter{
&loadbalancers.LBSpec{
Name: "api-server-lb",
Expand Down Expand Up @@ -2882,11 +3090,79 @@ func TestClusterScope_LBSpecs(t *testing.T) {
},
},
},
{
name: "Private API Server LB",
featureGate: feature.APIServerILB,
azureCluster: &infrav1.AzureCluster{
ObjectMeta: metav1.ObjectMeta{
Name: "my-cluster",
},
Spec: infrav1.AzureClusterSpec{
AzureClusterClassSpec: infrav1.AzureClusterClassSpec{
SubscriptionID: "123",
Location: "westus2",
},
ControlPlaneEnabled: true,
ResourceGroup: "my-rg",
NetworkSpec: infrav1.NetworkSpec{
Vnet: infrav1.VnetSpec{
Name: "my-vnet",
ResourceGroup: "my-rg",
},
Subnets: []infrav1.SubnetSpec{
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "cp-subnet",
Role: infrav1.SubnetControlPlane,
},
},
{
SubnetClassSpec: infrav1.SubnetClassSpec{
Name: "node-subnet",
Role: infrav1.SubnetNode,
},
},
},
APIServerLB: &infrav1.LoadBalancerSpec{
Name: "api-server-lb",
BackendPool: infrav1.BackendPool{
Name: "api-server-lb-backend-pool",
},
LoadBalancerClassSpec: infrav1.LoadBalancerClassSpec{
Type: infrav1.Internal,
IdleTimeoutInMinutes: ptr.To[int32](30),
SKU: infrav1.SKUStandard,
},
},
},
},
},
want: []azure.ResourceSpecGetter{
&loadbalancers.LBSpec{
Name: "api-server-lb",
ResourceGroup: "my-rg",
SubscriptionID: "123",
ClusterName: "my-cluster",
Location: "westus2",
VNetName: "my-vnet",
VNetResourceGroup: "my-rg",
SubnetName: "cp-subnet",
APIServerPort: 6443,
Type: infrav1.Internal,
SKU: infrav1.SKUStandard,
Role: infrav1.APIServerRole,
BackendPoolName: "api-server-lb-backend-pool",
IdleTimeoutInMinutes: ptr.To[int32](30),
AdditionalTags: infrav1.Tags{},
},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

if tc.featureGate == feature.APIServerILB {
defer featuregatetesting.SetFeatureGateDuringTest(t, feature.Gates, tc.featureGate, true)()
}
cluster := &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: tc.azureCluster.Name,
Expand Down
Loading
Loading