From e6daa47d5d314e3fde42f93a996bddcc2011d5ff Mon Sep 17 00:00:00 2001 From: lochan_2112 Date: Tue, 3 Oct 2023 17:44:37 +0530 Subject: [PATCH] Cherry pick PCP-1614 PCP-1615 (#96) * AKS UpgradeChannels (#92) * disableLocal accounts (#91) * disableLocal accounts * ensure to not downgrade an auto-upgraded cluster (#95) * ensure to not downgrade an auto-upgraded cluster * fix version issue for auto upgraded clusters (#97) --- azure/scope/managedcontrolplane.go | 77 +- azure/scope/managedcontrolplane_test.go | 745 +++++++++++++++++- azure/scope/managedmachinepool.go | 29 +- azure/scope/managedmachinepool_test.go | 130 ++- azure/services/agentpools/agentpools.go | 17 +- azure/services/managedclusters/client.go | 18 + .../managedclusters/managedclusters.go | 98 ++- .../managedclusters/managedclusters_test.go | 75 +- .../mock_managedclusters/client_mock.go | 15 + .../managedclusters_mock.go | 116 ++- azure/services/managedclusters/spec.go | 79 +- azure/services/managedclusters/spec_test.go | 18 +- azure/services/token/client.go | 80 ++ ...uster.x-k8s.io_azureclusteridentities.yaml | 12 +- ...er.x-k8s.io_azuremanagedcontrolplanes.yaml | 26 + config/default/manager_image_patch.yaml | 2 +- exp/api/v1alpha3/zz_generated.conversion.go | 3 + exp/api/v1alpha4/zz_generated.conversion.go | 3 + .../v1beta1/azuremanagedcontrolplane_types.go | 38 + .../azuremanagedcontrolplane_webhook.go | 101 ++- .../azuremanagedcontrolplane_webhook_test.go | 409 +++++++++- .../azuremanagedmachinepool_webhook_test.go | 3 +- exp/api/v1beta1/zz_generated.deepcopy.go | 25 + .../azuremanagedcontrolplane_reconciler.go | 30 +- go.mod | 25 +- go.sum | 52 +- .../prometheus/resources/bundle.yaml | 6 +- spectro/generated/core-base.yaml | 2 +- spectro/generated/core-global.yaml | 425 ++++++---- spectro/run.sh | 4 +- util/versions/version.go | 26 + util/versions/version_test.go | 100 +++ 32 files changed, 2448 insertions(+), 341 deletions(-) create mode 100644 azure/services/token/client.go create mode 100644 util/versions/version.go create mode 100644 util/versions/version_test.go diff --git a/azure/scope/managedcontrolplane.go b/azure/scope/managedcontrolplane.go index 9b5efb75a51..b9680ab79e6 100644 --- a/azure/scope/managedcontrolplane.go +++ b/azure/scope/managedcontrolplane.go @@ -106,10 +106,11 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane // ManagedControlPlaneScope defines the basic context for an actuator to operate upon. type ManagedControlPlaneScope struct { - Client client.Client - patchHelper *patch.Helper - kubeConfigData []byte - cache *ManagedControlPlaneCache + Client client.Client + patchHelper *patch.Helper + adminKubeConfigData []byte + userKubeConfigData []byte + cache *ManagedControlPlaneCache AzureClients Cluster *clusterv1.Cluster @@ -413,6 +414,43 @@ func (s *ManagedControlPlaneScope) ManagedClusterAnnotations() map[string]string return s.ControlPlane.Annotations } +// IsLocalAcountsDisabled checks if local accounts have been disabled. +func (s *ManagedControlPlaneScope) IsLocalAcountsDisabled() bool { + if s.IsAadEnabled() && + s.ControlPlane.Spec.DisableLocalAccounts != nil && + *s.ControlPlane.Spec.DisableLocalAccounts { + return true + } + return false +} + +// IsAadEnabled checks if aad is enabled. +func (s *ManagedControlPlaneScope) IsAadEnabled() bool { + if s.ControlPlane.Spec.AADProfile != nil && s.ControlPlane.Spec.AADProfile.Managed { + return true + } + return false +} + +// SetAutoUpgradeVersionStatus sets the auto upgrade version in status +func (s *ManagedControlPlaneScope) SetAutoUpgradeVersionStatus(version string) { + s.ControlPlane.Status.AutoUpgradeVersion = version +} + +// IsManagedVersionUpgrade checks if version is auto managed by AKS. +func (s *ManagedControlPlaneScope) IsManagedVersionUpgrade() bool { + return isManagedVersionUpgrade(s.ControlPlane) +} + +func isManagedVersionUpgrade(managedControlPlane *infrav1exp.AzureManagedControlPlane) bool { + if managedControlPlane.Spec.AutoUpgradeProfile != nil && + (managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1exp.UpgradeChannelNone && + managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != infrav1exp.UpgradeChannelNodeImage) { + return true + } + return false +} + // ManagedClusterSpec returns the managed cluster spec. func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure.ResourceSpecGetter { managedClusterSpec := managedclusters.ManagedClusterSpec{ @@ -477,6 +515,9 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure EnableAzureRBAC: s.ControlPlane.Spec.AADProfile.Managed, AdminGroupObjectIDs: s.ControlPlane.Spec.AADProfile.AdminGroupObjectIDs, } + if s.ControlPlane.Spec.DisableLocalAccounts != nil { + managedClusterSpec.DisableLocalAccounts = s.ControlPlane.Spec.DisableLocalAccounts + } } if s.ControlPlane.Spec.AddonProfiles != nil { @@ -522,6 +563,12 @@ func (s *ManagedControlPlaneScope) ManagedClusterSpec(ctx context.Context) azure } } + if s.ControlPlane.Spec.AutoUpgradeProfile != nil { + managedClusterSpec.AutoUpgradeProfile = &managedclusters.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: s.ControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel, + } + } + return &managedClusterSpec } @@ -573,14 +620,24 @@ func (s *ManagedControlPlaneScope) MakeEmptyKubeConfigSecret() corev1.Secret { } } -// GetKubeConfigData returns a []byte that contains kubeconfig. -func (s *ManagedControlPlaneScope) GetKubeConfigData() []byte { - return s.kubeConfigData +// GetAdminKubeConfigData returns admin kubeconfig. +func (s *ManagedControlPlaneScope) GetAdminKubeConfigData() []byte { + return s.adminKubeConfigData +} + +// SetAdminKubeConfigData sets adminKubeconfig data. +func (s *ManagedControlPlaneScope) SetAdminKubeConfigData(kubeConfigData []byte) { + s.adminKubeConfigData = kubeConfigData +} + +// GetUserKubeConfigData returns user kubeconfig, required when using AAD with AKS cluster. +func (s *ManagedControlPlaneScope) GetUserKubeConfigData() []byte { + return s.userKubeConfigData } -// SetKubeConfigData sets kubeconfig data. -func (s *ManagedControlPlaneScope) SetKubeConfigData(kubeConfigData []byte) { - s.kubeConfigData = kubeConfigData +// SetUserKubeConfigData sets userKubeconfig data. +func (s *ManagedControlPlaneScope) SetUserKubeConfigData(kubeConfigData []byte) { + s.userKubeConfigData = kubeConfigData } // SetLongRunningOperationState will set the future on the AzureManagedControlPlane status to allow the resource to continue diff --git a/azure/scope/managedcontrolplane_test.go b/azure/scope/managedcontrolplane_test.go index ebfebeac241..b8db1b0d4d5 100644 --- a/azure/scope/managedcontrolplane_test.go +++ b/azure/scope/managedcontrolplane_test.go @@ -21,10 +21,10 @@ import ( "testing" "github.com/Azure/go-autorest/autorest" - "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + pointer "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/managedclusters" infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" @@ -80,6 +80,7 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { Mode: "System", Cluster: "cluster1", VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: pointer.To("Linux"), }, }, }, @@ -118,9 +119,10 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) { SKU: "Standard_D2s_v3", Mode: "System", Replicas: 1, - Version: to.StringPtr("1.21.1"), + Version: pointer.To("1.21.1"), Cluster: "cluster1", VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: pointer.To("Linux"), }, }, }, @@ -267,3 +269,742 @@ func TestManagedControlPlaneScope_AddonProfiles(t *testing.T) { }) } } + +func TestManagedControlPlaneScope_AutoUpgradeProfile(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *managedclusters.ManagedClusterAutoUpgradeProfile + }{ + { + Name: "Without AutoUpgradeProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AutoUpgradeProfile UpgradeChannelNodeImage", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: &managedclusters.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.AutoUpgradeProfile).To(Equal(c.Expected)) + }) + } +} + +func TestManagedControlPlaneScope_AADProfile(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *managedclusters.AADProfile + }{ + { + Name: "Without AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AADProfile", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: &managedclusters.AADProfile{ + Managed: true, + EnableAzureRBAC: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.AADProfile).To(Equal(c.Expected)) + }) + } +} + +func TestManagedControlPlaneScope_DisableLocalAccounts(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected *bool + }{ + { + Name: "Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "Without AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: nil, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: pointer.To(true), + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + managedClusterGetter := s.ManagedClusterSpec(context.TODO()) + managedCluster, ok := managedClusterGetter.(*managedclusters.ManagedClusterSpec) + g.Expect(ok).To(BeTrue()) + g.Expect(managedCluster.DisableLocalAccounts).To(Equal(c.Expected)) + }) + } +} + +func TestIsAaDEnabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "AAD is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + aadEnabled := s.IsAadEnabled() + g.Expect(aadEnabled).To(Equal(c.Expected)) + }) + } +} + +func TestIsLocalAcountsDisabled(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "DisbaleLocalAccount is not enabled", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and Without DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "With AAdProfile and With DisableLocalAccounts", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + AADProfile: &infrav1.AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: pointer.To(true), + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + localAccountsDisabled := s.IsLocalAcountsDisabled() + g.Expect(localAccountsDisabled).To(Equal(c.Expected)) + }) + } +} + +func Test_IsManagedVersionUpgrade(t *testing.T) { + scheme := runtime.NewScheme() + _ = capiv1exp.AddToScheme(scheme) + _ = infrav1.AddToScheme(scheme) + + cases := []struct { + Name string + Input ManagedControlPlaneScopeParams + Expected bool + }{ + { + Name: "Upgrade channel not set", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + SubscriptionID: "00000000-0000-0000-0000-000000000000", + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "Upgradechannel is set rapid", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelRapid, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set patch", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelPatch, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set stable", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelStable, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: true, + }, + { + Name: "Upgradechannel is set none", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNone, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + { + Name: "Upgradechannel is set node-image", + Input: ManagedControlPlaneScopeParams{ + AzureClients: AzureClients{ + Authorizer: autorest.NullAuthorizer{}, + }, + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + }, + ControlPlane: &infrav1.AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster1", + Namespace: "default", + }, + Spec: infrav1.AzureManagedControlPlaneSpec{ + AutoUpgradeProfile: &infrav1.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: infrav1.UpgradeChannelNodeImage, + }, + }, + }, + ManagedMachinePools: []ManagedMachinePool{ + { + MachinePool: getMachinePool("pool0"), + InfraMachinePool: getAzureMachinePool("pool0", infrav1.NodePoolModeSystem), + }, + }, + }, + Expected: false, + }, + } + for _, c := range cases { + c := c + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(c.Input.ControlPlane).Build() + c.Input.Client = fakeClient + s, err := NewManagedControlPlaneScope(context.TODO(), c.Input) + g.Expect(err).To(Succeed()) + autoUpgrade := s.IsManagedVersionUpgrade() + g.Expect(autoUpgrade).To(Equal(c.Expected)) + }) + } +} diff --git a/azure/scope/managedmachinepool.go b/azure/scope/managedmachinepool.go index ae6ccd85961..1da1d3070df 100644 --- a/azure/scope/managedmachinepool.go +++ b/azure/scope/managedmachinepool.go @@ -23,11 +23,13 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/util/futures" "sigs.k8s.io/cluster-api-provider-azure/util/tele" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" clusterv1exp "sigs.k8s.io/cluster-api/exp/api/v1beta1" "sigs.k8s.io/cluster-api/util/conditions" @@ -135,11 +137,7 @@ func (s *ManagedMachinePoolScope) AgentPoolSpec() azure.AgentPoolSpec { func buildAgentPoolSpec(managedControlPlane *infrav1exp.AzureManagedControlPlane, machinePool *clusterv1exp.MachinePool, managedMachinePool *infrav1exp.AzureManagedMachinePool) azure.AgentPoolSpec { - var normalizedVersion *string - if machinePool.Spec.Template.Spec.Version != nil { - v := strings.TrimPrefix(*machinePool.Spec.Template.Spec.Version, "v") - normalizedVersion = &v - } + normalizedVersion := getManagedMachinePoolVersion(managedControlPlane, machinePool) replicas := int32(1) if machinePool.Spec.Replicas != nil { @@ -300,3 +298,24 @@ func (s *ManagedMachinePoolScope) UpdateCAPIMachinePoolAnnotations(ctx context.C func (s *ManagedMachinePoolScope) GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string { return s.MachinePool.Annotations[key] } + +// IsManagedVersionUpgrade checks if version is auto managed by AKS. +func (s *ManagedMachinePoolScope) IsManagedAutoUpgrade() bool { + return isManagedVersionUpgrade(s.ControlPlane) +} + +func getManagedMachinePoolVersion(managedControlPlane *infrav1exp.AzureManagedControlPlane, + machinePool *clusterv1exp.MachinePool) *string { + var v, av string + if machinePool != nil { + v = ptr.Deref(machinePool.Spec.Template.Spec.Version, "") + } + if managedControlPlane != nil { + av = managedControlPlane.Status.AutoUpgradeVersion + } + higherVersion, err := versions.GetHigherK8sVersion(v, av) + if err != nil { + return nil + } + return ptr.To(strings.TrimPrefix(higherVersion, "v")) +} diff --git a/azure/scope/managedmachinepool_test.go b/azure/scope/managedmachinepool_test.go index 907ca55a31a..dba02a8837c 100644 --- a/azure/scope/managedmachinepool_test.go +++ b/azure/scope/managedmachinepool_test.go @@ -25,6 +25,7 @@ import ( . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -66,7 +67,7 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -108,6 +109,7 @@ func TestManagedMachinePoolScope_Autoscaling(t *testing.T) { MinCount: to.Int32Ptr(2), MaxCount: to.Int32Ptr(10), VnetSubnetID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups//providers/Microsoft.Network/virtualNetworks//subnets/", + OSType: ptr.To("Linux"), }, }, } @@ -160,7 +162,7 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -195,6 +197,7 @@ func TestManagedMachinePoolScope_NodeLabels(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "System", @@ -256,6 +259,7 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -288,6 +292,7 @@ func TestManagedMachinePoolScope_MaxPods(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "System", @@ -347,7 +352,7 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ - + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -386,6 +391,7 @@ func TestManagedMachinePoolScope_Taints(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "User", @@ -445,6 +451,7 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool0", SKU: "Standard_D2s_v3", Replicas: 1, @@ -477,6 +484,7 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { }, }, Expected: azure.AgentPoolSpec{ + OSType: ptr.To("Linux"), Name: "pool1", SKU: "Standard_D2s_v3", Mode: "User", @@ -502,6 +510,122 @@ func TestManagedMachinePoolScope_OSDiskType(t *testing.T) { } } +func Test_getManagedMachinePoolVersion(t *testing.T) { + cases := []struct { + Name string + managedControlPlane *infrav1.AzureManagedControlPlane + machinePool *capiv1exp.MachinePool + Expected *string + }{ + { + Name: "Empty configs", + managedControlPlane: nil, + machinePool: nil, + Expected: nil, + }, + { + Name: "Empty mp", + managedControlPlane: &infrav1.AzureManagedControlPlane{}, + machinePool: nil, + Expected: nil, + }, + { + Name: "Only machine pool is available", + managedControlPlane: nil, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.15.0"), + }, + { + Name: "Only machine pool is available and cp is nil", + managedControlPlane: nil, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.15.0"), + }, + { + Name: "mcp.status.autoUpgradeVersion > mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.20.3"), + }, + { + Name: "suffix + mcp.status.autoUpgradeVersion > mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.15.0"), + }, + }, + }, + }, + Expected: ptr.To("1.20.3"), + }, + { + Name: "mcp.status.autoUpgradeVersion < mp.spec.template.spec.version", + managedControlPlane: &infrav1.AzureManagedControlPlane{ + Status: infrav1.AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.20.3", + }, + }, + machinePool: &capiv1exp.MachinePool{ + Spec: capiv1exp.MachinePoolSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + Version: ptr.To("v1.21.0"), + }, + }, + }, + }, + Expected: ptr.To("1.21.0"), + }, + } + + for _, c := range cases { + t.Run(c.Name, func(t *testing.T) { + g := NewWithT(t) + v := getManagedMachinePoolVersion(c.managedControlPlane, c.machinePool) + if c.Expected != nil { + g.Expect(*v).To(Equal(*c.Expected)) + } else { + g.Expect(v).To(Equal(c.Expected)) + } + }) + } +} + func getAzureMachinePool(name string, mode infrav1.NodePoolMode) *infrav1.AzureManagedMachinePool { return &infrav1.AzureManagedMachinePool{ ObjectMeta: metav1.ObjectMeta{ diff --git a/azure/services/agentpools/agentpools.go b/azure/services/agentpools/agentpools.go index 8953586c921..4664d80851c 100644 --- a/azure/services/agentpools/agentpools.go +++ b/azure/services/agentpools/agentpools.go @@ -25,11 +25,13 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" "sigs.k8s.io/cluster-api-provider-azure/util/maps" "sigs.k8s.io/cluster-api-provider-azure/util/tele" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" ) const serviceName = "agentpools" @@ -47,6 +49,7 @@ type ManagedMachinePoolScope interface { UpdateCAPIMachinePoolReplicas(ctx context.Context, replicas *int32) UpdateCAPIMachinePoolAnnotations(ctx context.Context, key, value string) GetCAPIMachinePoolAnnotation(ctx context.Context, key string) string + IsManagedAutoUpgrade() bool } // Service provides operations on Azure resources. @@ -78,7 +81,6 @@ func (s *Service) Reconcile(ctx context.Context) error { agentPoolSpec := s.scope.AgentPoolSpec() profile := converters.AgentPoolToContainerServiceAgentPool(agentPoolSpec) - existingPool, err := s.Client.Get(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name) if err != nil && !azure.ResourceNotFound(err) { return errors.Wrap(err, "failed to get existing agent pool") @@ -91,6 +93,9 @@ func (s *Service) Reconcile(ctx context.Context) error { customHeaders := maps.FilterByKeyPrefix(s.scope.AgentPoolAnnotations(), azure.CustomHeaderPrefix) if isCreate := azure.ResourceNotFound(err); isCreate { + if s.scope.IsManagedAutoUpgrade() { + profile.OrchestratorVersion = nil + } err = s.Client.CreateOrUpdate(ctx, agentPoolSpec.ResourceGroup, agentPoolSpec.Cluster, agentPoolSpec.Name, profile, customHeaders) if err != nil && azure.ResourceNotFound(err) { @@ -105,6 +110,14 @@ func (s *Service) Reconcile(ctx context.Context) error { log.V(2).Info(msg) return azure.WithTransientError(errors.New(msg), 20*time.Second) } + // Get the higher version out of the existing and new version + profileOrchestratorVersion, err := versions.GetHigherK8sVersion( + ptr.Deref(existingPool.OrchestratorVersion, ""), + ptr.Deref(profile.OrchestratorVersion, "")) + + if err != nil { + return errors.Wrap(err, "error while calculating k8s version") + } // Normalize individual agent pools to diff in case we need to update existingProfile := containerservice.AgentPool{ @@ -122,7 +135,7 @@ func (s *Service) Reconcile(ctx context.Context) error { normalizedProfile := containerservice.AgentPool{ ManagedClusterAgentPoolProfileProperties: &containerservice.ManagedClusterAgentPoolProfileProperties{ Count: profile.Count, - OrchestratorVersion: profile.OrchestratorVersion, + OrchestratorVersion: ptr.To(profileOrchestratorVersion), Mode: profile.Mode, EnableAutoScaling: profile.EnableAutoScaling, MinCount: profile.MinCount, diff --git a/azure/services/managedclusters/client.go b/azure/services/managedclusters/client.go index 73ab07b1448..6f6ca0ccb7d 100644 --- a/azure/services/managedclusters/client.go +++ b/azure/services/managedclusters/client.go @@ -33,6 +33,7 @@ import ( // CredentialGetter is a helper interface for getting managed cluster credentials. type CredentialGetter interface { GetCredentials(context.Context, string, string) ([]byte, error) + GetUserCredentials(context.Context, string, string) ([]byte, error) } // azureClient contains the Azure go-sdk Client. @@ -79,6 +80,23 @@ func (ac *azureClient) GetCredentials(ctx context.Context, resourceGroupName, na return *(*credentialList.Kubeconfigs)[0].Value, nil } +// GetUserCredentials fetches the user kubeconfig for a managed cluster. +func (ac *azureClient) GetUserCredentials(ctx context.Context, resourceGroupName, name string) ([]byte, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "managedclusters.azureClient.GetCredentials") + defer done() + + credentialList, err := ac.managedclusters.ListClusterUserCredentials(ctx, resourceGroupName, name, "") + if err != nil { + return nil, err + } + + if credentialList.Kubeconfigs == nil || len(*credentialList.Kubeconfigs) < 1 { + return nil, errors.New("no user kubeconfigs available for the managed cluster") + } + + return *(*credentialList.Kubeconfigs)[0].Value, nil +} + // CreateOrUpdateAsync creates or updates a managed cluster. // It sends a PUT request to Azure and if accepted without error, the func will return a Future which can be used to track the ongoing // progress of the operation. diff --git a/azure/services/managedclusters/managedclusters.go b/azure/services/managedclusters/managedclusters.go index b9662600a67..4902558a5e2 100644 --- a/azure/services/managedclusters/managedclusters.go +++ b/azure/services/managedclusters/managedclusters.go @@ -18,20 +18,26 @@ package managedclusters import ( "context" + "fmt" "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" "github.com/Azure/go-autorest/autorest/to" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/clientcmd" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/services/async" + "sigs.k8s.io/cluster-api-provider-azure/azure/services/token" "sigs.k8s.io/cluster-api-provider-azure/util/reconciler" "sigs.k8s.io/cluster-api-provider-azure/util/tele" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) -const serviceName = "managedcluster" +const ( + serviceName = "managedcluster" + aadResourceId = "6dae42f8-4368-4678-94ff-3960e28e3630" +) // ManagedClusterScope defines the scope interface for a managed cluster. type ManagedClusterScope interface { @@ -40,8 +46,14 @@ type ManagedClusterScope interface { ManagedClusterSpec(context.Context) azure.ResourceSpecGetter SetControlPlaneEndpoint(clusterv1.APIEndpoint) MakeEmptyKubeConfigSecret() corev1.Secret - GetKubeConfigData() []byte - SetKubeConfigData([]byte) + GetAdminKubeConfigData() []byte + SetAdminKubeConfigData([]byte) + GetUserKubeConfigData() []byte + SetUserKubeConfigData([]byte) + IsAadEnabled() bool + IsLocalAcountsDisabled() bool + SetAutoUpgradeVersionStatus(version string) + IsManagedVersionUpgrade() bool } // Service provides operations on azure resources. @@ -94,11 +106,18 @@ func (s *Service) Reconcile(ctx context.Context) error { // Update kubeconfig data // Always fetch credentials in case of rotation - kubeConfigData, err := s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + adminKubeConfigData, userKubeConfigData, err := s.ReconcileKubeconfig(ctx, managedClusterSpec) if err != nil { - return errors.Wrap(err, "failed to get credentials for managed cluster") + return errors.Wrap(err, "error while reconciling adminKubeConfigData") + } + + if s.Scope.IsManagedVersionUpgrade() && managedCluster.KubernetesVersion != nil { + kubernetesVersion := fmt.Sprintf("v%s", *managedCluster.KubernetesVersion) + s.Scope.SetAutoUpgradeVersionStatus(kubernetesVersion) } - s.Scope.SetKubeConfigData(kubeConfigData) + + s.Scope.SetAdminKubeConfigData(adminKubeConfigData) + s.Scope.SetUserKubeConfigData(userKubeConfigData) } s.Scope.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, resultErr) return resultErr @@ -126,3 +145,70 @@ func (s *Service) Delete(ctx context.Context) error { func (s *Service) IsManaged(ctx context.Context) (bool, error) { return true, nil } + +func (s *Service) ReconcileKubeconfig(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, []byte, error) { + var ( + userKubeConfigData []byte + adminKubeConfigData []byte + err error + ) + + if s.Scope.IsAadEnabled() { + if userKubeConfigData, err = s.GetUserKubeConfigData(ctx, managedClusterSpec); err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig") + } + } + + if s.Scope.IsLocalAcountsDisabled() { + userKubeconfigWithToken, err := s.GetUserKubeConfigWithToken(userKubeConfigData, ctx, managedClusterSpec) + if err != nil { + return nil, nil, errors.Wrap(err, "error while trying to get user kubeconfig with token") + } + return userKubeconfigWithToken, userKubeConfigData, nil + } + + adminKubeConfigData, err = s.GetCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return adminKubeConfigData, userKubeConfigData, nil +} + +func (s *Service) GetUserKubeConfigData(ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + kubeConfigData, err := s.GetUserCredentials(ctx, managedClusterSpec.ResourceGroupName(), managedClusterSpec.ResourceName()) + if err != nil { + return nil, errors.Wrap(err, "failed to get credentials for managed cluster") + } + return kubeConfigData, nil +} + +func (s *Service) GetUserKubeConfigWithToken(userKubeConfigData []byte, ctx context.Context, managedClusterSpec azure.ResourceSpecGetter) ([]byte, error) { + + tokenClient, err := token.NewClient(s.Scope) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token client") + } + + token, err := tokenClient.GetAzureActiveDirectoryToken(ctx, aadResourceId) + if err != nil { + return nil, errors.Wrap(err, "error while getting aad token for user kubeconfig") + } + + return s.CreateUserKubeconfigWithToken(token, userKubeConfigData) +} + +func (s *Service) CreateUserKubeconfigWithToken(token string, userKubeConfigData []byte) ([]byte, error) { + config, err := clientcmd.Load(userKubeConfigData) + if err != nil { + return nil, errors.Wrap(err, "error while trying to unmarshal new user kubeconfig with token") + } + for _, auth := range config.AuthInfos { + auth.Token = token + auth.Exec = nil + } + kubeconfig, err := clientcmd.Write(*config) + if err != nil { + return nil, errors.Wrap(err, "error while trying to marshal new user kubeconfig with token") + } + return kubeconfig, nil +} diff --git a/azure/services/managedclusters/managedclusters_test.go b/azure/services/managedclusters/managedclusters_test.go index af875091dd6..549e380902f 100644 --- a/azure/services/managedclusters/managedclusters_test.go +++ b/azure/services/managedclusters/managedclusters_test.go @@ -33,6 +33,14 @@ import ( ) var fakeManagedClusterSpec = &ManagedClusterSpec{Name: "my-managedcluster", ResourceGroup: "my-rg"} +var fakeManagedClusterSpecWithAAD = &ManagedClusterSpec{ + Name: "my-managedcluster", + ResourceGroup: "my-rg", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"000000-000000-000000-000000"}, + }, +} func TestReconcile(t *testing.T) { testcases := []struct { @@ -60,6 +68,7 @@ func TestReconcile(t *testing.T) { name: "create managed cluster succeeds", expectedError: "", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + var userKubeConfigData []byte s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ @@ -71,14 +80,74 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) + m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData(userKubeConfigData) + s.IsManagedVersionUpgrade().Return(false) + s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) + }, + }, + { + name: "create managed cluster succeeds, update autoupgrade status", + expectedError: "", + expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + var userKubeConfigData []byte + s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) + r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ + ManagedClusterProperties: &containerservice.ManagedClusterProperties{ + Fqdn: pointer.String("my-managedcluster-fqdn"), + ProvisioningState: pointer.String("Succeeded"), + KubernetesVersion: pointer.String("1.27.3"), + }, + }, nil) + s.SetControlPlaneEndpoint(clusterv1.APIEndpoint{ + Host: "my-managedcluster-fqdn", + Port: 443, + }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) + m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData(userKubeConfigData) + s.IsManagedVersionUpgrade().Return(true) + s.SetAutoUpgradeVersionStatus("v1.27.3") + s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) + }, + }, + { + name: "create managed cluster succeeds with user kubeconfig", + expectedError: "", + expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { + s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpecWithAAD) + r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpecWithAAD, serviceName).Return(containerservice.ManagedCluster{ + ManagedClusterProperties: &containerservice.ManagedClusterProperties{ + Fqdn: pointer.String("my-managedcluster-fqdn"), + ProvisioningState: pointer.String("Succeeded"), + AadProfile: &containerservice.ManagedClusterAADProfile{ + Managed: pointer.Bool(true), + AdminGroupObjectIDs: &[]string{"000000-000000-000000-000000"}, + }, + }, + }, nil) + s.SetControlPlaneEndpoint(clusterv1.APIEndpoint{ + Host: "my-managedcluster-fqdn", + Port: 443, + }) + s.IsAadEnabled().Return(true) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials"), nil) - s.SetKubeConfigData([]byte("credentials")) + m.GetUserCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte("credentials-user"), nil) + s.SetAdminKubeConfigData([]byte("credentials")) + s.SetUserKubeConfigData([]byte("credentials-user")) + s.IsManagedVersionUpgrade().Return(false) s.UpdatePutStatus(infrav1.ManagedClusterRunningCondition, serviceName, nil) }, }, { name: "fail to get managed cluster credentials", - expectedError: "failed to get credentials for managed cluster: internal server error", + expectedError: "error while reconciling adminKubeConfigData: failed to get credentials for managed cluster: internal server error", expect: func(m *mock_managedclusters.MockCredentialGetterMockRecorder, s *mock_managedclusters.MockManagedClusterScopeMockRecorder, r *mock_async.MockReconcilerMockRecorder) { s.ManagedClusterSpec(gomockinternal.AContext()).Return(fakeManagedClusterSpec) r.CreateResource(gomockinternal.AContext(), fakeManagedClusterSpec, serviceName).Return(containerservice.ManagedCluster{ @@ -91,6 +160,8 @@ func TestReconcile(t *testing.T) { Host: "my-managedcluster-fqdn", Port: 443, }) + s.IsAadEnabled().Return(false) + s.IsLocalAcountsDisabled().Return(false) m.GetCredentials(gomockinternal.AContext(), "my-rg", "my-managedcluster").Return([]byte(""), errors.New("internal server error")) }, }, diff --git a/azure/services/managedclusters/mock_managedclusters/client_mock.go b/azure/services/managedclusters/mock_managedclusters/client_mock.go index 735939603c5..77216af037c 100644 --- a/azure/services/managedclusters/mock_managedclusters/client_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/client_mock.go @@ -64,3 +64,18 @@ func (mr *MockCredentialGetterMockRecorder) GetCredentials(arg0, arg1, arg2 inte mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetCredentials), arg0, arg1, arg2) } + +// GetUserCredentials mocks base method. +func (m *MockCredentialGetter) GetUserCredentials(arg0 context.Context, arg1, arg2 string) ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserCredentials", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUserCredentials indicates an expected call of GetUserCredentials. +func (mr *MockCredentialGetterMockRecorder) GetUserCredentials(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserCredentials", reflect.TypeOf((*MockCredentialGetter)(nil).GetUserCredentials), arg0, arg1, arg2) +} diff --git a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go index abf3c16817b..1badc3806c1 100644 --- a/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go +++ b/azure/services/managedclusters/mock_managedclusters/managedclusters_mock.go @@ -137,18 +137,18 @@ func (mr *MockManagedClusterScopeMockRecorder) DeleteLongRunningOperationState(a return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).DeleteLongRunningOperationState), arg0, arg1) } -// GetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) GetKubeConfigData() []byte { +// GetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetAdminKubeConfigData() []byte { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetKubeConfigData") + ret := m.ctrl.Call(m, "GetAdminKubeConfigData") ret0, _ := ret[0].([]byte) return ret0 } -// GetKubeConfigData indicates an expected call of GetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) GetKubeConfigData() *gomock.Call { +// GetAdminKubeConfigData indicates an expected call of GetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetAdminKubeConfigData() *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetKubeConfigData)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetAdminKubeConfigData)) } // GetLongRunningOperationState mocks base method. @@ -165,6 +165,20 @@ func (mr *MockManagedClusterScopeMockRecorder) GetLongRunningOperationState(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).GetLongRunningOperationState), arg0, arg1) } +// GetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) GetUserKubeConfigData() []byte { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUserKubeConfigData") + ret0, _ := ret[0].([]byte) + return ret0 +} + +// GetUserKubeConfigData indicates an expected call of GetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) GetUserKubeConfigData() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).GetUserKubeConfigData)) +} + // HashKey mocks base method. func (m *MockManagedClusterScope) HashKey() string { m.ctrl.T.Helper() @@ -179,6 +193,48 @@ func (mr *MockManagedClusterScopeMockRecorder) HashKey() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HashKey", reflect.TypeOf((*MockManagedClusterScope)(nil).HashKey)) } +// IsAadEnabled mocks base method. +func (m *MockManagedClusterScope) IsAadEnabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsAadEnabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsAadEnabled indicates an expected call of IsAadEnabled. +func (mr *MockManagedClusterScopeMockRecorder) IsAadEnabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsAadEnabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsAadEnabled)) +} + +// IsLocalAcountsDisabled mocks base method. +func (m *MockManagedClusterScope) IsLocalAcountsDisabled() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsLocalAcountsDisabled") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsLocalAcountsDisabled indicates an expected call of IsLocalAcountsDisabled. +func (mr *MockManagedClusterScopeMockRecorder) IsLocalAcountsDisabled() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsLocalAcountsDisabled", reflect.TypeOf((*MockManagedClusterScope)(nil).IsLocalAcountsDisabled)) +} + +// IsManagedVersionUpgrade mocks base method. +func (m *MockManagedClusterScope) IsManagedVersionUpgrade() bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsManagedVersionUpgrade") + ret0, _ := ret[0].(bool) + return ret0 +} + +// IsManagedVersionUpgrade indicates an expected call of IsManagedVersionUpgrade. +func (mr *MockManagedClusterScopeMockRecorder) IsManagedVersionUpgrade() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsManagedVersionUpgrade", reflect.TypeOf((*MockManagedClusterScope)(nil).IsManagedVersionUpgrade)) +} + // MakeEmptyKubeConfigSecret mocks base method. func (m *MockManagedClusterScope) MakeEmptyKubeConfigSecret() v1.Secret { m.ctrl.T.Helper() @@ -207,28 +263,40 @@ func (mr *MockManagedClusterScopeMockRecorder) ManagedClusterSpec(arg0 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ManagedClusterSpec", reflect.TypeOf((*MockManagedClusterScope)(nil).ManagedClusterSpec), arg0) } -// SetControlPlaneEndpoint mocks base method. -func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { +// SetAdminKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetAdminKubeConfigData(arg0 []byte) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) + m.ctrl.Call(m, "SetAdminKubeConfigData", arg0) } -// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. -func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { +// SetAdminKubeConfigData indicates an expected call of SetAdminKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetAdminKubeConfigData(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAdminKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAdminKubeConfigData), arg0) +} + +// SetAutoUpgradeVersionStatus mocks base method. +func (m *MockManagedClusterScope) SetAutoUpgradeVersionStatus(version string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetAutoUpgradeVersionStatus", version) +} + +// SetAutoUpgradeVersionStatus indicates an expected call of SetAutoUpgradeVersionStatus. +func (mr *MockManagedClusterScopeMockRecorder) SetAutoUpgradeVersionStatus(version interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetAutoUpgradeVersionStatus", reflect.TypeOf((*MockManagedClusterScope)(nil).SetAutoUpgradeVersionStatus), version) } -// SetKubeConfigData mocks base method. -func (m *MockManagedClusterScope) SetKubeConfigData(arg0 []byte) { +// SetControlPlaneEndpoint mocks base method. +func (m *MockManagedClusterScope) SetControlPlaneEndpoint(arg0 v1beta10.APIEndpoint) { m.ctrl.T.Helper() - m.ctrl.Call(m, "SetKubeConfigData", arg0) + m.ctrl.Call(m, "SetControlPlaneEndpoint", arg0) } -// SetKubeConfigData indicates an expected call of SetKubeConfigData. -func (mr *MockManagedClusterScopeMockRecorder) SetKubeConfigData(arg0 interface{}) *gomock.Call { +// SetControlPlaneEndpoint indicates an expected call of SetControlPlaneEndpoint. +func (mr *MockManagedClusterScopeMockRecorder) SetControlPlaneEndpoint(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetKubeConfigData), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetControlPlaneEndpoint", reflect.TypeOf((*MockManagedClusterScope)(nil).SetControlPlaneEndpoint), arg0) } // SetLongRunningOperationState mocks base method. @@ -243,6 +311,18 @@ func (mr *MockManagedClusterScopeMockRecorder) SetLongRunningOperationState(arg0 return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLongRunningOperationState", reflect.TypeOf((*MockManagedClusterScope)(nil).SetLongRunningOperationState), arg0) } +// SetUserKubeConfigData mocks base method. +func (m *MockManagedClusterScope) SetUserKubeConfigData(arg0 []byte) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetUserKubeConfigData", arg0) +} + +// SetUserKubeConfigData indicates an expected call of SetUserKubeConfigData. +func (mr *MockManagedClusterScopeMockRecorder) SetUserKubeConfigData(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetUserKubeConfigData", reflect.TypeOf((*MockManagedClusterScope)(nil).SetUserKubeConfigData), arg0) +} + // SubscriptionID mocks base method. func (m *MockManagedClusterScope) SubscriptionID() string { m.ctrl.T.Helper() diff --git a/azure/services/managedclusters/spec.go b/azure/services/managedclusters/spec.go index d0aa605dd7e..1d62a19b765 100644 --- a/azure/services/managedclusters/spec.go +++ b/azure/services/managedclusters/spec.go @@ -26,10 +26,12 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/google/go-cmp/cmp" "github.com/pkg/errors" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" "sigs.k8s.io/cluster-api-provider-azure/azure/converters" expinfrav1 "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" ) // ManagedClusterSpec contains properties to create a managed cluster. @@ -80,7 +82,7 @@ type ManagedClusterSpec struct { ServiceCIDR string // DockerBridgeCidr - A CIDR notation IP range assigned to the Docker bridge network. It must not overlap with any Subnet IP ranges or the Kubernetes service address range. - DockerBridgeCidr *string `json:"dockerBridgeCidr,omitempty"` + DockerBridgeCidr *string // DNSServiceIP is an IP address assigned to the Kubernetes DNS service DNSServiceIP *string @@ -111,6 +113,18 @@ type ManagedClusterSpec struct { // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster UserAssignedIdentities []UserAssignedIdentity + + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + DisableLocalAccounts *bool + + // AutoUpgradeProfile - Profile of auto upgrade configuration. + AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile +} + +// ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. +type ManagedClusterAutoUpgradeProfile struct { + // UpgradeChannel - upgrade channel for auto upgrade. Possible values include: 'UpgradeChannelRapid', 'UpgradeChannelStable', 'UpgradeChannelPatch', 'UpgradeChannelNodeImage', 'UpgradeChannelNone' + UpgradeChannel expinfrav1.UpgradeChannel } // UserAssignedIdentity defines the user-assigned identities provided @@ -118,7 +132,7 @@ type ManagedClusterSpec struct { type UserAssignedIdentity struct { // ProviderID is the identification ID of the user-assigned Identity, the format of an identity is: // 'azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}' - ProviderID string `json:"providerID"` + ProviderID string } // AADProfile is Azure Active Directory configuration to integrate with AKS, for aad authentication. @@ -203,6 +217,28 @@ func (s *ManagedClusterSpec) CustomHeaders() map[string]string { return s.Headers } +// GetManagedClusterVersion gets the desired managed k8s version. +// If autoupgrade channels is set to patch, stable or rapid, clusters can be upgraded to higher version by AKS. +// If autoupgrade is triggered, existing kubernetes version will be higher than the user desired kubernetes version. +// CAPZ should honour the upgrade and it should not downgrade to the lower desired version. +func (s *ManagedClusterSpec) GetManagedClusterVersion(existing interface{}) (string, error) { + version := s.Version + if existing != nil && version != "" { + existingMC, ok := existing.(containerservice.ManagedCluster) + if !ok { + return version, fmt.Errorf("%T is not a containerservice.ManagedCluster", existing) + } + if v, err := versions.GetHigherK8sVersion( + version, + ptr.Deref(existingMC.KubernetesVersion, version)); err != nil { + return "", err + } else { + version = v + } + } + return version, nil +} + // Parameters returns the parameters for the managed clusters. func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{}, err error) { decodedSSHPublicKey, err := base64.StdEncoding.DecodeString(s.SSHPublicKey) @@ -218,7 +254,6 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ NodeResourceGroup: &s.NodeResourceGroup, EnableRBAC: to.BoolPtr(true), DNSPrefix: s.DNSPrefix, - KubernetesVersion: &s.Version, LinuxProfile: &containerservice.LinuxProfile{ AdminUsername: to.StringPtr(azure.DefaultAKSUserName), SSH: &containerservice.SSHConfiguration{ @@ -241,6 +276,12 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ }, } + if kubernetesVersion, err := s.GetManagedClusterVersion(existing); err != nil { + return nil, err + } else { + managedCluster.KubernetesVersion = &kubernetesVersion + } + if s.FqdnSubdomain != nil { managedCluster.FqdnSubdomain = s.FqdnSubdomain } @@ -282,6 +323,9 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ EnableAzureRBAC: &s.AADProfile.EnableAzureRBAC, AdminGroupObjectIDs: &s.AADProfile.AdminGroupObjectIDs, } + if s.DisableLocalAccounts != nil { + managedCluster.DisableLocalAccounts = s.DisableLocalAccounts + } } for i := range s.AddonProfiles { @@ -355,6 +399,12 @@ func (s *ManagedClusterSpec) Parameters(existing interface{}) (params interface{ } } + if s.AutoUpgradeProfile != nil { + managedCluster.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: containerservice.UpgradeChannel(s.AutoUpgradeProfile.UpgradeChannel), + } + } + if existing != nil { existingMC, ok := existing.(containerservice.ManagedCluster) if !ok { @@ -536,6 +586,29 @@ func computeDiffOfNormalizedClusters(managedCluster containerservice.ManagedClus } } + if managedCluster.AutoUpgradeProfile != nil { + clusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: managedCluster.AutoUpgradeProfile.UpgradeChannel, + } + } + + if existingMC.AutoUpgradeProfile != nil { + if existingMC.AutoUpgradeProfile.UpgradeChannel == "" { + existingMC.AutoUpgradeProfile = nil + } else { + existingMCClusterNormalized.AutoUpgradeProfile = &containerservice.ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: existingMC.AutoUpgradeProfile.UpgradeChannel, + } + } + } + if managedCluster.DisableLocalAccounts != nil { + clusterNormalized.DisableLocalAccounts = managedCluster.DisableLocalAccounts + } + + if existingMC.DisableLocalAccounts != nil { + existingMCClusterNormalized.DisableLocalAccounts = existingMC.DisableLocalAccounts + } + diff := cmp.Diff(clusterNormalized, existingMCClusterNormalized) return diff } diff --git a/azure/services/managedclusters/spec_test.go b/azure/services/managedclusters/spec_test.go index e6256e900a0..0347d9f4107 100644 --- a/azure/services/managedclusters/spec_test.go +++ b/azure/services/managedclusters/spec_test.go @@ -110,6 +110,23 @@ func TestParameters(t *testing.T) { g.Expect(result).To(BeNil()) }, }, + { + name: "managedcluster exists, no update needed", + existing: getExistingCluster(), + spec: &ManagedClusterSpec{ + Name: "test-managedcluster", + ResourceGroup: "test-rg", + Location: "test-location", + Tags: map[string]string{ + "test-tag": "test-value", + }, + Version: "v1.21.0", + LoadBalancerSKU: "Standard", + }, + expect: func(g *WithT, result interface{}) { + g.Expect(result).To(BeNil()) + }, + }, { name: "managedcluster exists and an update is needed", existing: getExistingCluster(), @@ -158,7 +175,6 @@ func getSampleManagedCluster() containerservice.ManagedCluster { return containerservice.ManagedCluster{ ManagedClusterProperties: &containerservice.ManagedClusterProperties{ KubernetesVersion: to.StringPtr("v1.22.0"), - DNSPrefix: to.StringPtr("test-managedcluster"), AgentPoolProfiles: &[]containerservice.ManagedClusterAgentPoolProfile{ converters.AgentPoolToManagedClusterAgentPoolProfile(azure.AgentPoolSpec{ Name: "test-agentpool-0", diff --git a/azure/services/token/client.go b/azure/services/token/client.go new file mode 100644 index 00000000000..23f6e473cab --- /dev/null +++ b/azure/services/token/client.go @@ -0,0 +1,80 @@ +package token + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + az "github.com/Azure/go-autorest/autorest/azure" + "github.com/pkg/errors" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "sigs.k8s.io/cluster-api-provider-azure/util/tele" +) + +const ( + defaultEnvironmentName = "AzurePublicCloud" +) + +type AzureClient struct { + aadToken *azidentity.ClientSecretCredential +} + +// newClient creates a new managed cluster client from an authorizer. +func NewClient(auth azure.Authorizer) (*AzureClient, error) { + aadToken, err := newAzureActiveDirectoryTokenClient(auth.TenantID(), + auth.ClientID(), + auth.ClientSecret(), + auth.CloudEnvironment()) + if err != nil { + return nil, err + } + return &AzureClient{ + aadToken: aadToken, + }, nil +} + +// newAzureActiveDirectoryTokenClient creates a new aad token client from an authorizer. +func newAzureActiveDirectoryTokenClient(tenantId, clientId, clientSecret, envName string) (*azidentity.ClientSecretCredential, error) { + cliOpts, err := getAzureClientOptions(envName) + if err != nil { + return nil, errors.Wrap(err, "error while getting client options") + } + clientOptions := &azidentity.ClientSecretCredentialOptions{ + ClientOptions: cliOpts, + } + cred, err := azidentity.NewClientSecretCredential(tenantId, clientId, clientSecret, clientOptions) + if err != nil { + return nil, errors.Wrap(err, "error while getting az client secret credentials") + } + return cred, nil +} + +func getAzureClientOptions(environment string) (azcore.ClientOptions, error) { + + if environment == "" { + environment = defaultEnvironmentName + } + env, err := az.EnvironmentFromName(environment) + if err != nil { + return azcore.ClientOptions{}, errors.Wrap(err, "error while getting azure env") + } + c := cloud.Configuration{ + ActiveDirectoryAuthorityHost: env.ActiveDirectoryEndpoint, + } + return azcore.ClientOptions{ + Cloud: c, + }, nil +} + +func (ac *AzureClient) GetAzureActiveDirectoryToken(ctx context.Context, resourceId string) (string, error) { + ctx, _, done := tele.StartSpanWithLogger(ctx, "aadToken.GetToken") + defer done() + + spnAccessToken, err := ac.aadToken.GetToken(ctx, policy.TokenRequestOptions{Scopes: []string{resourceId + "/.default"}}) + if err != nil { + return "", errors.Wrap(err, "failed to get token") + } + return spnAccessToken.Token, nil +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml index 22517fe9ac5..ceefbf2f571 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusteridentities.yaml @@ -56,11 +56,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -233,11 +233,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -411,11 +411,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml index 254468b334a..5ab7c8e8ac0 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml @@ -577,6 +577,22 @@ spec: cluster. type: string type: object + autoUpgradeProfile: + description: AutoUpgradeProfile - Profile of auto upgrade configuration. + properties: + upgradeChannel: + description: 'UpgradeChannel - upgrade channel for auto upgrade. + Possible values include: "node-image","none","patch","rapid","stable"' + enum: + - node-image + - none + - patch + - rapid + - stable + type: string + required: + - upgradeChannel + type: object controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to communicate with the control plane. @@ -592,6 +608,11 @@ spec: - host - port type: object + disableLocalAccounts: + description: DisableLocalAccounts - If set to true, getting static + credential will be disabled for this cluster. Expected to only be + used for AAD clusters. + type: boolean dnsPrefix: description: DNSPrefix - DNS prefix specified when creating the managed cluster. @@ -799,6 +820,11 @@ spec: description: AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. properties: + autoUpgradeVersion: + description: AutoUpgradeVersion is the Kubernetes version populated + after autoupgrade based on the upgrade channel. + minLength: 2 + type: string conditions: description: Conditions defines current service state of the AzureManagedControlPlane. items: diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml index f8141cfee55..4242ef31345 100644 --- a/config/default/manager_image_patch.yaml +++ b/config/default/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/spectro-dev-public/devop2023/release-fips/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev + - image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev name: manager diff --git a/exp/api/v1alpha3/zz_generated.conversion.go b/exp/api/v1alpha3/zz_generated.conversion.go index 454de50d771..f2af706016e 100644 --- a/exp/api/v1alpha3/zz_generated.conversion.go +++ b/exp/api/v1alpha3/zz_generated.conversion.go @@ -772,6 +772,8 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha3_AzureManagedCo // WARNING: in.LoadBalancerProfile requires manual conversion: does not exist in peer-type // WARNING: in.APIServerAccessProfile requires manual conversion: does not exist in peer-type // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type + // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type + // WARNING: in.DisableLocalAccounts requires manual conversion: does not exist in peer-type return nil } @@ -787,6 +789,7 @@ func Convert_v1alpha3_AzureManagedControlPlaneStatus_To_v1beta1_AzureManagedCont } func autoConvert_v1beta1_AzureManagedControlPlaneStatus_To_v1alpha3_AzureManagedControlPlaneStatus(in *v1beta1.AzureManagedControlPlaneStatus, out *AzureManagedControlPlaneStatus, s conversion.Scope) error { + // WARNING: in.AutoUpgradeVersion requires manual conversion: does not exist in peer-type out.Ready = in.Ready out.Initialized = in.Initialized // WARNING: in.Conditions requires manual conversion: does not exist in peer-type diff --git a/exp/api/v1alpha4/zz_generated.conversion.go b/exp/api/v1alpha4/zz_generated.conversion.go index c5fc195c2e6..3ed27aae260 100644 --- a/exp/api/v1alpha4/zz_generated.conversion.go +++ b/exp/api/v1alpha4/zz_generated.conversion.go @@ -1075,6 +1075,8 @@ func autoConvert_v1beta1_AzureManagedControlPlaneSpec_To_v1alpha4_AzureManagedCo out.LoadBalancerProfile = (*LoadBalancerProfile)(unsafe.Pointer(in.LoadBalancerProfile)) out.APIServerAccessProfile = (*APIServerAccessProfile)(unsafe.Pointer(in.APIServerAccessProfile)) // WARNING: in.UserAssignedIdentities requires manual conversion: does not exist in peer-type + // WARNING: in.AutoUpgradeProfile requires manual conversion: does not exist in peer-type + // WARNING: in.DisableLocalAccounts requires manual conversion: does not exist in peer-type return nil } @@ -1091,6 +1093,7 @@ func Convert_v1alpha4_AzureManagedControlPlaneStatus_To_v1beta1_AzureManagedCont } func autoConvert_v1beta1_AzureManagedControlPlaneStatus_To_v1alpha4_AzureManagedControlPlaneStatus(in *v1beta1.AzureManagedControlPlaneStatus, out *AzureManagedControlPlaneStatus, s conversion.Scope) error { + // WARNING: in.AutoUpgradeVersion requires manual conversion: does not exist in peer-type out.Ready = in.Ready out.Initialized = in.Initialized // WARNING: in.Conditions requires manual conversion: does not exist in peer-type diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_types.go b/exp/api/v1beta1/azuremanagedcontrolplane_types.go index c7582e0484f..4f8afccfb33 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_types.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_types.go @@ -45,6 +45,22 @@ const ( ManagedControlPlaneOutboundTypeUserDefinedRouting ManagedControlPlaneOutboundType = "userDefinedRouting" ) +// UpgradeChannel enumerates the values for upgrade channel. +type UpgradeChannel string + +const ( + // UpgradeChannelNodeImage ... + UpgradeChannelNodeImage UpgradeChannel = "node-image" + // UpgradeChannelNone ... + UpgradeChannelNone UpgradeChannel = "none" + // UpgradeChannelPatch ... + UpgradeChannelPatch UpgradeChannel = "patch" + // UpgradeChannelRapid ... + UpgradeChannelRapid UpgradeChannel = "rapid" + // UpgradeChannelStable ... + UpgradeChannelStable UpgradeChannel = "stable" +) + // AzureManagedControlPlaneSpec defines the desired state of AzureManagedControlPlane. type AzureManagedControlPlaneSpec struct { // Version defines the desired Kubernetes version. @@ -147,6 +163,22 @@ type AzureManagedControlPlaneSpec struct { // UserAssignedIdentities is a list of standalone Azure identities provided by the user to assign the cluster // +optional UserAssignedIdentities []infrav1.UserAssignedIdentity `json:"userAssignedIdentities,omitempty"` + + // AutoUpgradeProfile - Profile of auto upgrade configuration. + // +optional + AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile `json:"autoUpgradeProfile,omitempty"` + + // DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters. + // +optional + DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"` +} + +// ManagedClusterAutoUpgradeProfile auto upgrade profile for a managed cluster. +type ManagedClusterAutoUpgradeProfile struct { + // UpgradeChannel - upgrade channel for auto upgrade. Possible values include: "node-image","none","patch","rapid","stable" + // +kubebuilder:validation:Enum=node-image;none;patch;rapid;stable + // +kubebuilder:validation:Required + UpgradeChannel UpgradeChannel `json:"upgradeChannel"` } // AADProfile - AAD integration managed by AKS. @@ -251,6 +283,12 @@ type ManagedControlPlaneSubnet struct { // AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. type AzureManagedControlPlaneStatus struct { + + // AutoUpgradeVersion is the Kubernetes version populated after autoupgrade based on the upgrade channel. + // +kubebuilder:validation:MinLength:=2 + // +optional + AutoUpgradeVersion string `json:"autoUpgradeVersion,omitempty"` + // Ready is true when the provider resource is ready. // +optional Ready bool `json:"ready,omitempty"` diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go index eaf2f570e12..6d397a37821 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook.go @@ -30,8 +30,9 @@ import ( "k8s.io/apimachinery/pkg/runtime" kerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" + "sigs.k8s.io/cluster-api-provider-azure/util/versions" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -160,6 +161,16 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if old.Spec.AutoUpgradeProfile != nil && m.Spec.AutoUpgradeProfile == nil { + // Prevent AutoUpgradeProfile to be set to nil. + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "AutoUpgradeProfile"), + m.Spec.AutoUpgradeProfile, + "field is cannot be set to nil, to disable auto upgrades set the channel to none.")) + } + if old.Spec.NetworkPlugin != nil { // Prevent NetworkPlugin modification if it was already set to some value if m.Spec.NetworkPlugin == nil { @@ -242,6 +253,34 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } + if m.Spec.DisableLocalAccounts != nil && + m.Spec.AADProfile == nil { + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "DisableLocalAccounts can be set only for AAD enabled clusters")) + } + + if old.Spec.DisableLocalAccounts != nil { + // Prevent DisableLocalAccounts modification if it was already set to some value + if m.Spec.DisableLocalAccounts == nil { + // unsetting the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + m.Spec.DisableLocalAccounts, + "field is immutable, unsetting is not allowed")) + } else if *m.Spec.DisableLocalAccounts != *old.Spec.DisableLocalAccounts { + // changing the field is not allowed + allErrs = append(allErrs, + field.Invalid( + field.NewPath("Spec", "DisableLocalAccounts"), + *m.Spec.DisableLocalAccounts, + "field is immutable")) + } + } + if old.Spec.OutboundType != nil { // Prevent OutboundType modification if it was already set to some value if m.Spec.OutboundType == nil { @@ -261,7 +300,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client } } - if pointer.StringDeref(m.Spec.DNSPrefix, "") != pointer.StringDeref(old.Spec.DNSPrefix, "") { + if ptr.Deref[string](m.Spec.DNSPrefix, "") != ptr.Deref[string](old.Spec.DNSPrefix, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.DNSPrefix"), @@ -269,7 +308,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } - if pointer.StringDeref(m.Spec.DockerBridgeCidr, "") != pointer.StringDeref(old.Spec.DockerBridgeCidr, "") { + if ptr.Deref[string](m.Spec.DockerBridgeCidr, "") != ptr.Deref[string](old.Spec.DockerBridgeCidr, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.DockerBridgeCidr"), @@ -277,7 +316,7 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } - if pointer.StringDeref(m.Spec.FqdnSubdomain, "") != pointer.StringDeref(old.Spec.FqdnSubdomain, "") { + if ptr.Deref[string](m.Spec.FqdnSubdomain, "") != ptr.Deref[string](old.Spec.FqdnSubdomain, "") { allErrs = append(allErrs, field.Invalid( field.NewPath("Spec.FqdnSubdomain"), @@ -285,6 +324,20 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client "field is immutable")) } + if hv, _ := versions.GetHigherK8sVersion(m.Spec.Version, old.Spec.Version); hv != m.Spec.Version { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Version"), + m.Spec.Version, "fields version cannot be downgraded"), + ) + } + + if old.Status.AutoUpgradeVersion != "" && m.Spec.Version != old.Spec.Version { + if hv, _ := versions.GetHigherK8sVersion(m.Spec.Version, old.Status.AutoUpgradeVersion); hv != m.Spec.Version { + allErrs = append(allErrs, field.Invalid(field.NewPath("Spec", "Version"), + m.Spec.Version, "fields version cannot be downgraded"), + ) + } + } + if errs := m.validateAPIServerAccessProfileUpdate(old); len(errs) > 0 { allErrs = append(allErrs, errs...) } @@ -311,6 +364,7 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { m.validateLoadBalancerProfile, m.validateAPIServerAccessProfile, m.validateDNSPrefix, + m.validateDisableLocalAccounts, //m.validateManagedClusterNetwork, } @@ -324,8 +378,16 @@ func (m *AzureManagedControlPlane) Validate(cli client.Client) error { return kerrors.NewAggregate(errs) } -func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { +func (m *AzureManagedControlPlane) validateDisableLocalAccounts(_ client.Client) error { + + if m.Spec.DisableLocalAccounts != nil && m.Spec.AADProfile == nil { + return errors.New("DisableLocalAccounts should be set only for AAD enabled clusters") + } + return nil +} + +func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { if m.Spec.DNSPrefix == nil { return nil } @@ -335,32 +397,12 @@ func (m *AzureManagedControlPlane) validateDNSPrefix(_ client.Client) error { // 2. Alphanumerics and hyphens: [a-zA-Z0-9-] // 3. Start and end with alphanumeric: ^[a-zA-Z0-9].*[a-zA-Z0-9]$ regex := regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9-]{0,52}[a-zA-Z0-9]$`) - if regex.MatchString(pointer.StringDeref(m.Spec.DNSPrefix, "")) { + if regex.MatchString(ptr.Deref[string](m.Spec.DNSPrefix, "")) { return nil } return errors.New("DNSPrefix is invalid") } -// func (m *AzureManagedControlPlane) validateFqdnSubdomain(_ client.Client) error { - -// if m.Spec.FqdnSubdomain == nil { -// return nil -// } - -// // Regex pattern for FQDN subdomain validation -// // 1. Between 1 and 63 characters long: {1,63} -// // 2. Alphanumerics and hyphens. -// // 3. Start and end with alphanumeric. -// // 4. Parts separated by dots (.) -// pattern := `^(?i)[a-z0-9]([a-z0-9-]*[a-z0-9])?(?:\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*(?:\.aks\.example)$` - -// regex := regexp.MustCompile(pattern) -// if regex.MatchString(pointer.StringDeref(m.Spec.FqdnSubdomain, "")) { -// return nil -// } -// return errors.Errorf("FqdnSubdomain is invalid %s", pointer.StringDeref(m.Spec.FqdnSubdomain, "")) -// } - // validateDNSServiceIP validates the DNSServiceIP. func (m *AzureManagedControlPlane) validateDNSServiceIP(_ client.Client) error { if m.Spec.DNSServiceIP != nil { @@ -389,7 +431,6 @@ func (m *AzureManagedControlPlane) validateVersion(_ client.Client) error { if !kubeSemver.MatchString(m.Spec.Version) { return errors.New("must be a valid semantic version") } - return nil } @@ -573,10 +614,10 @@ func (m *AzureManagedControlPlane) validateAPIServerAccessProfileDNSZoneUpdate(o // You can only update from byo or system to none. No other combination of update values is supported. if m.Spec.APIServerAccessProfile != nil && old.Spec.APIServerAccessProfile != nil && - pointer.BoolDeref(m.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && - pointer.BoolDeref(old.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + ptr.Deref(m.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && + ptr.Deref(old.Spec.APIServerAccessProfile.EnablePrivateCluster, false) && m.Spec.APIServerAccessProfile.PrivateDNSZone != old.Spec.APIServerAccessProfile.PrivateDNSZone && - pointer.StringDeref(m.Spec.APIServerAccessProfile.PrivateDNSZone, "") == "None" { + ptr.Deref[string](m.Spec.APIServerAccessProfile.PrivateDNSZone, "") == "None" { return nil } diff --git a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go index 07e34697670..c5481550ddf 100644 --- a/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedcontrolplane_webhook_test.go @@ -22,7 +22,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestDefaultingWebhook(t *testing.T) { @@ -83,10 +83,10 @@ func TestValidatingWebhook(t *testing.T) { expectErr bool }{ { - name: "Testing inValid DNSPrefix for starting with invalid charecters", + name: "Testing inValid DNSPrefix for starting with invalid characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("-thisi$"), + DNSPrefix: ptr.To("-thisi$"), Version: "v1.17.8", }, }, @@ -96,7 +96,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing inValid DNSPrefix with more then 54 characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("thisisaverylong$^clusternameconsistingofmorethan54characterswhichshouldbeinvalid"), + DNSPrefix: ptr.To("thisisaverylong$^clusternameconsistingofmorethan54characterswhichshouldbeinvalid"), Version: "v1.17.8", }, }, @@ -106,7 +106,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing inValid DNSPrefix with underscore", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("no_underscore"), + DNSPrefix: ptr.To("no_underscore"), Version: "v1.17.8", }, }, @@ -116,17 +116,37 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing inValid DNSPrefix with special characters", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("no-dollar$@%"), + DNSPrefix: ptr.To("no-dollar$@%"), Version: "v1.17.8", }, }, expectErr: true, }, + { + name: "Testing Valid DNSPrefix with hyphen characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("hyphen-allowed"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, + { + name: "Testing Valid DNSPrefix with hyphen characters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSPrefix: ptr.To("palette-test07"), + Version: "v1.17.8", + }, + }, + expectErr: false, + }, { name: "Testing valid DNSPrefix ", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("thisisavlerylongclu7l0sternam3leconsistingofmorethan54"), + DNSPrefix: ptr.To("thisisavlerylongclu7l0sternam3leconsistingofmorethan54"), Version: "v1.17.8", }, }, @@ -136,7 +156,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing valid DNSServiceIP", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "v1.17.8", }, }, @@ -146,7 +166,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Testing invalid DNSServiceIP", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0.3"), + DNSServiceIP: ptr.To("192.168.0.0.3"), Version: "v1.17.8", }, }, @@ -156,7 +176,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Invalid Version", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "honk", }, }, @@ -166,7 +186,7 @@ func TestValidatingWebhook(t *testing.T) { name: "not following the Kubernetes Version pattern", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "1.19.0", }, }, @@ -176,7 +196,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Version not set", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "", }, }, @@ -186,7 +206,7 @@ func TestValidatingWebhook(t *testing.T) { name: "Valid Version", amcp: AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSServiceIP: pointer.StringPtr("192.168.0.0"), + DNSServiceIP: ptr.To("192.168.0.0"), Version: "v1.17.8", }, }, @@ -284,6 +304,30 @@ func TestValidatingWebhook(t *testing.T) { }, expectErr: true, }, + { + name: "DisableLocalAccounts cannot be set for non AAD clusters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + DisableLocalAccounts: ptr.To(true), + }, + }, + expectErr: true, + }, + { + name: "DisableLocalAccounts can be set for AAD clusters", + amcp: AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.21.2", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(true), + }, + }, + expectErr: false, + }, } for _, tt := range tests { @@ -427,6 +471,117 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { amcp: createAzureManagedControlPlane("192.168.0.0", "1.999.9", generateSSHPublicKey(true)), wantErr: true, }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.17.0", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "v1.18.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.1", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane invalid version downgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.6", + }, + }, + wantErr: true, + }, + { + name: "AzureManagedControlPlane no version change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane valid version upgrade change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.19.5", + }, + }, + wantErr: false, + }, + { + name: "AzureManagedControlPlane valid version change", + oldAMCP: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.18.0", + }, + Status: AzureManagedControlPlaneStatus{ + AutoUpgradeVersion: "1.19.3", + }, + }, + amcp: &AzureManagedControlPlane{ + Spec: AzureManagedControlPlaneSpec{ + DNSServiceIP: to.StringPtr("192.168.0.0"), + Version: "v1.19.3", + }, + }, + wantErr: false, + }, { name: "AzureManagedControlPlane SubscriptionID is immutable", oldAMCP: &AzureManagedControlPlane{ @@ -833,13 +988,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane DNSPrefix is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks-1"), + DNSPrefix: ptr.To("capz-aks-1"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks"), + DNSPrefix: ptr.To("capz-aks"), Version: "v1.18.0", }, }, @@ -849,13 +1004,13 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane DNSPrefix is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks"), + DNSPrefix: ptr.To("capz-aks"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - DNSPrefix: pointer.StringPtr("capz-aks"), + DNSPrefix: ptr.To("capz-aks"), Version: "v1.18.0", }, }, @@ -865,14 +1020,14 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane FqdnSubdomain is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - FqdnSubdomain: pointer.StringPtr("capzaks.api"), + FqdnSubdomain: ptr.To("capzaks.api"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", - FqdnSubdomain: pointer.StringPtr("capzaks.com"), + FqdnSubdomain: ptr.To("capzaks.com"), }, }, wantErr: true, @@ -881,14 +1036,14 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { name: "AzureManagedControlPlane FqdnSubdomain is immutable", oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ - FqdnSubdomain: pointer.StringPtr("capzaks.api"), + FqdnSubdomain: ptr.To("capzaks.api"), Version: "v1.18.0", }, }, amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", - FqdnSubdomain: pointer.StringPtr("capzaks.api"), + FqdnSubdomain: ptr.To("capzaks.api"), }, }, wantErr: false, @@ -898,7 +1053,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { oldAMCP: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), + EnablePrivateCluster: ptr.To(true), }, Version: "v1.18.0", }, @@ -906,8 +1061,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { amcp: &AzureManagedControlPlane{ Spec: AzureManagedControlPlaneSpec{ APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("None"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("None"), }, Version: "v1.18.0", }, @@ -920,8 +1075,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), }, }, }, @@ -929,8 +1084,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("None"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("None"), }, }, }, @@ -942,8 +1097,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), }, }, }, @@ -951,7 +1106,7 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), + EnablePrivateCluster: ptr.To(true), }, }, }, @@ -963,8 +1118,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id"), }, }, }, @@ -972,8 +1127,8 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { Spec: AzureManagedControlPlaneSpec{ Version: "v1.18.0", APIServerAccessProfile: &APIServerAccessProfile{ - EnablePrivateCluster: pointer.Bool(true), - PrivateDNSZone: pointer.StringPtr("example-resource-id-1"), + EnablePrivateCluster: ptr.To(true), + PrivateDNSZone: ptr.To("example-resource-id-1"), }, }, }, @@ -1023,6 +1178,190 @@ func TestAzureManagedControlPlane_ValidateUpdate(t *testing.T) { }, wantErr: true, }, + { + name: "DisableLocalAccounts can be set only for AAD enabled clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: ptr.To(true), + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + wantErr: false, + }, + { + name: "DisableLocalAccounts cannot be set only for non AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + DisableLocalAccounts: ptr.To(true), + }, + }, + wantErr: true, + }, + { + name: "DisableLocalAccounts cannot be disabled AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(true), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + }, + }, + wantErr: true, + }, + { + name: "DisableLocalAccounts cannot change the value AAD clusters", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(true), + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AADProfile: &AADProfile{ + Managed: true, + AdminGroupObjectIDs: []string{"00000000-0000-0000-0000-000000000000"}, + }, + DisableLocalAccounts: ptr.To(false), + }, + }, + wantErr: true, + }, + { + name: "Auto upgrade profile cannot be disabled", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelNone, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + }, + }, + wantErr: true, + }, + { + name: "Auto upgrade profile channel can be updated", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelStable, + }, + }, + }, + wantErr: false, + }, + { + name: "Auto upgrade profile channel can remain same", + oldAMCP: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + amcp: &AzureManagedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + }, + Spec: AzureManagedControlPlaneSpec{ + Version: "v1.18.0", + AutoUpgradeProfile: &ManagedClusterAutoUpgradeProfile{ + UpgradeChannel: UpgradeChannelPatch, + }, + }, + }, + wantErr: false, + }, { name: "OutboundType update", oldAMCP: &AzureManagedControlPlane{ diff --git a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go index dda97a621c0..5c17db43360 100644 --- a/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go +++ b/exp/api/v1beta1/azuremanagedmachinepool_webhook_test.go @@ -17,9 +17,10 @@ limitations under the License. package v1beta1 import ( - "sigs.k8s.io/cluster-api-provider-azure/azure" "testing" + "sigs.k8s.io/cluster-api-provider-azure/azure" + "github.com/Azure/azure-sdk-for-go/services/containerservice/mgmt/2021-05-01/containerservice" "github.com/Azure/go-autorest/autorest/to" . "github.com/onsi/gomega" diff --git a/exp/api/v1beta1/zz_generated.deepcopy.go b/exp/api/v1beta1/zz_generated.deepcopy.go index e90218f9258..84656a0a7f2 100644 --- a/exp/api/v1beta1/zz_generated.deepcopy.go +++ b/exp/api/v1beta1/zz_generated.deepcopy.go @@ -711,6 +711,16 @@ func (in *AzureManagedControlPlaneSpec) DeepCopyInto(out *AzureManagedControlPla *out = make([]apiv1beta1.UserAssignedIdentity, len(*in)) copy(*out, *in) } + if in.AutoUpgradeProfile != nil { + in, out := &in.AutoUpgradeProfile, &out.AutoUpgradeProfile + *out = new(ManagedClusterAutoUpgradeProfile) + **out = **in + } + if in.DisableLocalAccounts != nil { + in, out := &in.DisableLocalAccounts, &out.DisableLocalAccounts + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AzureManagedControlPlaneSpec. @@ -983,6 +993,21 @@ func (in *MachineRollingUpdateDeployment) DeepCopy() *MachineRollingUpdateDeploy return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ManagedClusterAutoUpgradeProfile) DeepCopyInto(out *ManagedClusterAutoUpgradeProfile) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ManagedClusterAutoUpgradeProfile. +func (in *ManagedClusterAutoUpgradeProfile) DeepCopy() *ManagedClusterAutoUpgradeProfile { + if in == nil { + return nil + } + out := new(ManagedClusterAutoUpgradeProfile) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ManagedControlPlaneSubnet) DeepCopyInto(out *ManagedControlPlaneSubnet) { *out = *in diff --git a/exp/controllers/azuremanagedcontrolplane_reconciler.go b/exp/controllers/azuremanagedcontrolplane_reconciler.go index afc0b301d38..26fbba56688 100644 --- a/exp/controllers/azuremanagedcontrolplane_reconciler.go +++ b/exp/controllers/azuremanagedcontrolplane_reconciler.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "fmt" "github.com/pkg/errors" "sigs.k8s.io/cluster-api-provider-azure/azure" @@ -92,20 +93,25 @@ func (r *azureManagedControlPlaneService) reconcileKubeconfig(ctx context.Contex ctx, _, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedControlPlaneService.reconcileKubeconfig") defer done() - kubeConfigData := r.scope.GetKubeConfigData() - if kubeConfigData == nil { - return nil - } - kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + kubeConfigs := [][]byte{r.scope.GetAdminKubeConfigData(), r.scope.GetUserKubeConfigData()} - // Always update credentials in case of rotation - if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { - kubeConfigSecret.Data = map[string][]byte{ - secret.KubeconfigDataName: kubeConfigData, + for i, kubeConfigData := range kubeConfigs { + if len(kubeConfigData) == 0 { + continue + } + kubeConfigSecret := r.scope.MakeEmptyKubeConfigSecret() + if i == 1 { + // 2nd kubeconfig is the user kubeconfig + kubeConfigSecret.Name = fmt.Sprintf("%s-user", kubeConfigSecret.Name) + } + if _, err := controllerutil.CreateOrUpdate(ctx, r.kubeclient, &kubeConfigSecret, func() error { + kubeConfigSecret.Data = map[string][]byte{ + secret.KubeconfigDataName: kubeConfigData, + } + return nil + }); err != nil { + return errors.Wrap(err, "failed to kubeconfig secret for cluster") } - return nil - }); err != nil { - return errors.Wrap(err, "failed to kubeconfig secret for cluster") } return nil diff --git a/go.mod b/go.mod index 1e47d0bae7c..86dda86eed5 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.19 require ( github.com/Azure/aad-pod-identity v1.8.13 - github.com/Azure/azure-sdk-for-go v63.4.0+incompatible + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 github.com/Azure/go-autorest/autorest v0.11.27 github.com/Azure/go-autorest/autorest/adal v0.9.20 github.com/Azure/go-autorest/autorest/azure/auth v0.5.10 @@ -32,16 +34,16 @@ require ( go.opentelemetry.io/otel/sdk v1.4.0 go.opentelemetry.io/otel/sdk/metric v0.27.0 go.opentelemetry.io/otel/trace v1.4.0 - golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 + golang.org/x/crypto v0.12.0 + golang.org/x/mod v0.8.0 helm.sh/helm/v3 v3.8.1 k8s.io/api v0.25.0 k8s.io/apimachinery v0.25.0 k8s.io/client-go v0.25.0 k8s.io/component-base v0.25.0 - k8s.io/klog/v2 v2.70.1 + k8s.io/klog/v2 v2.80.1 k8s.io/kubectl v0.25.0 - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed + k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/cluster-api v1.1.1 sigs.k8s.io/cluster-api/test v1.1.4 sigs.k8s.io/controller-runtime v0.13.1 @@ -49,12 +51,14 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -102,6 +106,7 @@ require ( github.com/gofrs/uuid v4.2.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.1 // indirect @@ -123,6 +128,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.13.6 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/lib/pq v1.10.4 // indirect @@ -151,6 +157,7 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect @@ -176,12 +183,12 @@ require ( go.opentelemetry.io/otel/internal/metric v0.27.0 // indirect go.opentelemetry.io/proto/otlp v0.12.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index fd34e40b4d1..37d6030d3a1 100644 --- a/go.sum +++ b/go.sum @@ -52,8 +52,14 @@ github.com/Azure/aad-pod-identity v1.8.13 h1:/gUmacA0z7+lsOlGYAYzkGvAB/KOkUe5Pb6 github.com/Azure/aad-pod-identity v1.8.13/go.mod h1:uxM/lsPo/abzqdk0rwEm4SqO9pavMz0fCmKpYAj4HL8= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v63.4.0+incompatible h1:fle3M5Q7vr8auaiPffKyUQmLbvYeqpw30bKU6PrWJFo= -github.com/Azure/azure-sdk-for-go v63.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= @@ -93,6 +99,8 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU= @@ -385,6 +393,7 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/ github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684 h1:DBZ2sN7CK6dgvHVpQsQj4sRMCbWTmd17l+5SUCjnQSY= github.com/distribution/distribution/v3 v3.0.0-20211118083504-a29a3c99a684/go.mod h1:UfCu3YXJJCI+IdnqGgYP82dk2+Joxmv+mUTVBES6wac= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v20.10.11+incompatible h1:tXU1ezXcruZQRrMP8RN2z9N91h+6egZTS1gsPsKantc= github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= @@ -546,6 +555,8 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -790,6 +801,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw= github.com/lann/builder v0.0.0-20180802200727-47ae307949d0/go.mod h1:dXGbAdH5GtBTC4WfIxhKZfyBF/HBFgRZSWwZ9g/He9o= github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 h1:P6pPBnrTSX3DEVR4fDembhRWSsG5rVo6hYhAB/ADZrk= @@ -978,6 +991,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1273,8 +1288,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd h1:XcWmESyNjXJMLahc3mqVQJcgSTDxFxhETVlfk9uGc38= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1311,8 +1326,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1373,8 +1388,8 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1504,6 +1519,7 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1520,13 +1536,13 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1536,8 +1552,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1903,8 +1919,8 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw= @@ -1920,8 +1936,8 @@ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.1.0 h1:tfWM1RT7PzUwWphqHU6ptPU3ZhwVnSw/9nEGf519rYg= oras.land/oras-go v1.1.0/go.mod h1:1A7vR/0KknT2UkJVWh+xMi95I/AhK8ZrxrnUSmXN0bQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/hack/observability/prometheus/resources/bundle.yaml b/hack/observability/prometheus/resources/bundle.yaml index 2fce683d225..e65100bf1f3 100644 --- a/hack/observability/prometheus/resources/bundle.yaml +++ b/hack/observability/prometheus/resources/bundle.yaml @@ -4486,7 +4486,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: @@ -11335,7 +11335,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: @@ -17146,7 +17146,7 @@ spec: If underlying persistent volume is being resized then the Condition will be set to 'ResizeStarted'. items: - description: PersistentVolumeClaimCondition contails + description: PersistentVolumeClaimCondition contains details about state of pvc properties: lastProbeTime: diff --git a/spectro/generated/core-base.yaml b/spectro/generated/core-base.yaml index ab8d2dee686..1ee34adc537 100644 --- a/spectro/generated/core-base.yaml +++ b/spectro/generated/core-base.yaml @@ -61,7 +61,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 + image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev imagePullPolicy: Always name: manager terminationGracePeriodSeconds: 10 diff --git a/spectro/generated/core-global.yaml b/spectro/generated/core-global.yaml index f5215b48c86..18148548248 100644 --- a/spectro/generated/core-global.yaml +++ b/spectro/generated/core-global.yaml @@ -11,6 +11,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -77,11 +78,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -254,11 +255,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -432,11 +433,11 @@ spec: either a Service Principal password or certificate secret. properties: name: - description: Name is unique within a namespace to reference a + description: name is unique within a namespace to reference a secret resource. type: string namespace: - description: Namespace defines the space within which the secret + description: namespace defines the space within which the secret name must be unique. type: string type: object @@ -528,6 +529,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -2218,6 +2220,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -2272,6 +2276,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -2324,6 +2330,8 @@ spec: the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string name: type: string sku: @@ -2688,6 +2696,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -3033,6 +3042,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -3052,6 +3063,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -3069,6 +3082,8 @@ spec: for the TCP idle connection. format: int32 type: integer + ipAllocationMethod: + type: string sku: description: SKU defines an Azure load balancer SKU. type: string @@ -3436,6 +3451,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -3924,6 +3940,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -6119,6 +6136,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -7711,6 +7729,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -8952,6 +8971,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -9141,6 +9161,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -9726,10 +9747,23 @@ spec: privateDNSZone: description: PrivateDNSZone - Private dns zone mode for private cluster. + type: string + type: object + autoUpgradeProfile: + description: AutoUpgradeProfile - Profile of auto upgrade configuration. + properties: + upgradeChannel: + description: 'UpgradeChannel - upgrade channel for auto upgrade. + Possible values include: "node-image","none","patch","rapid","stable"' enum: - - System - - None + - node-image + - none + - patch + - rapid + - stable type: string + required: + - upgradeChannel type: object controlPlaneEndpoint: description: ControlPlaneEndpoint represents the endpoint used to @@ -9746,11 +9780,29 @@ spec: - host - port type: object + disableLocalAccounts: + description: DisableLocalAccounts - If set to true, getting static + credential will be disabled for this cluster. Expected to only be + used for AAD clusters. + type: boolean + dnsPrefix: + description: DNSPrefix - DNS prefix specified when creating the managed + cluster. + type: string dnsServiceIP: description: DNSServiceIP is an IP address assigned to the Kubernetes DNS service. It must be within the Kubernetes service address range specified in serviceCidr. type: string + dockerBridgeCidr: + description: DockerBridgeCidr - A CIDR notation IP range assigned + to the Docker bridge network. It must not overlap with any Subnet + IP ranges or the Kubernetes service address range. + type: string + fqdnSubdomain: + description: FqdnSubdomain - FQDN subdomain specified when creating + private cluster with custom private dns zone. + type: string identityRef: description: IdentityRef is a reference to a AzureClusterIdentity to be used when reconciling this cluster @@ -9851,6 +9903,14 @@ spec: containing cluster IaaS resources. Will be populated to default in webhook. type: string + outboundType: + description: Outbound configuration used by Nodes. + enum: + - loadBalancer + - managedNATGateway + - userAssignedNATGateway + - userDefinedRouting + type: string resourceGroupName: description: ResourceGroupName is the name of the Azure resource group for this AKS Cluster. @@ -9875,6 +9935,21 @@ spec: description: SubscriptionID is the GUID of the Azure subscription to hold this cluster. type: string + userAssignedIdentities: + description: UserAssignedIdentities is a list of standalone Azure + identities provided by the user to assign the cluster + items: + description: UserAssignedIdentity defines the user-assigned identities + provided by the user to be assigned to Azure resources. + properties: + providerID: + description: 'ProviderID is the identification ID of the user-assigned + Identity, the format of an identity is: ''azure:///subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{identityName}''' + type: string + required: + - providerID + type: object + type: array version: description: Version defines the desired Kubernetes version. minLength: 2 @@ -9917,6 +9992,11 @@ spec: description: AzureManagedControlPlaneStatus defines the observed state of AzureManagedControlPlane. properties: + autoUpgradeVersion: + description: AutoUpgradeVersion is the Kubernetes version populated + after autoupgrade based on the upgrade channel. + minLength: 2 + type: string conditions: description: Conditions defines current service state of the AzureManagedControlPlane. items: @@ -10027,6 +10107,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert controller-gen.kubebuilder.io/version: v0.8.0 + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure cluster.x-k8s.io/v1beta1: v1beta1 @@ -10531,6 +10612,169 @@ status: conditions: [] storedVersions: [] --- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + creationTimestamp: null + labels: + cluster.x-k8s.io/provider: infrastructure-azure + name: capz-mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azurecluster.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclusters + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azureclustertemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azureclustertemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachine.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachines + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremachinetemplate.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinetemplates + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool + failurePolicy: Fail + name: default.azuremachinepool.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremachinepools + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane + failurePolicy: Fail + name: default.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedcontrolplanes + sideEffects: None +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: capz-webhook-service + namespace: capi-webhook-system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool + failurePolicy: Fail + matchPolicy: Equivalent + name: default.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - azuremanagedmachinepools + sideEffects: None +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -10656,7 +10900,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: gcr.io/spectro-dev-public/snehal/cluster-api-azure/cluster-api-azure-controller:spectro-v1.3.2-20221102 + image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev imagePullPolicy: Always livenessProbe: httpGet: @@ -10805,172 +11049,11 @@ spec: selfSigned: {} --- apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - annotations: - cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert - labels: - cluster.x-k8s.io/provider: infrastructure-azure - name: capz-mutating-webhook-configuration -webhooks: -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azurecluster - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azurecluster.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azureclusters - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azureclustertemplate - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azureclustertemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azureclustertemplates - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachine - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azuremachine.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremachines - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinetemplate - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azuremachinetemplate.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremachinetemplates - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremachinepool - failurePolicy: Fail - name: default.azuremachinepool.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremachinepools - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane - failurePolicy: Fail - name: default.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremanagedcontrolplanes - sideEffects: None -- admissionReviewVersions: - - v1 - - v1beta1 - clientConfig: - service: - name: capz-webhook-service - namespace: capi-webhook-system - path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedmachinepool - failurePolicy: Fail - matchPolicy: Equivalent - name: default.azuremanagedmachinepools.infrastructure.cluster.x-k8s.io - rules: - - apiGroups: - - infrastructure.cluster.x-k8s.io - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - azuremanagedmachinepools - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: annotations: cert-manager.io/inject-ca-from: capi-webhook-system/capz-serving-cert + creationTimestamp: null labels: cluster.x-k8s.io/provider: infrastructure-azure name: capz-validating-webhook-configuration diff --git a/spectro/run.sh b/spectro/run.sh index 31df3583833..95acb764911 100755 --- a/spectro/run.sh +++ b/spectro/run.sh @@ -2,5 +2,5 @@ rm generated/* -kustomize build --load-restrictor LoadRestrictionsNone global > ./generated/core-global.yaml -kustomize build --load-restrictor LoadRestrictionsNone base > ./generated/core-base.yaml +kustomize build --load_restrictor none global > ./generated/core-global.yaml +kustomize build --load_restrictor none base > ./generated/core-base.yaml diff --git a/util/versions/version.go b/util/versions/version.go new file mode 100644 index 00000000000..4646c3c8cfe --- /dev/null +++ b/util/versions/version.go @@ -0,0 +1,26 @@ +package versions + +import ( + semverv4 "github.com/blang/semver" + "github.com/pkg/errors" +) + +// GetHigherK8sVersion returns the higher k8s version out of a and b. +func GetHigherK8sVersion(a, b string) (string, error) { + v1, errv1 := semverv4.ParseTolerant(a) + v2, errv2 := semverv4.ParseTolerant(b) + if errv1 != nil && errv2 != nil { + return "", errors.Wrapf(errv1, "error parsing k8s version %s, %v error parsing k8s version %s", a, errv2, b) + } + if errv1 != nil { + return b, nil + } + if errv2 != nil { + return a, nil + } + + if v1.GTE(v2) { + return a, nil + } + return b, nil +} diff --git a/util/versions/version_test.go b/util/versions/version_test.go new file mode 100644 index 00000000000..738b39e6ba4 --- /dev/null +++ b/util/versions/version_test.go @@ -0,0 +1,100 @@ +package versions + +import ( + "testing" + + . "github.com/onsi/gomega" +) + +func TestGetHigherK8sVersion(t *testing.T) { + cases := []struct { + name string + a string + b string + output string + expectErr bool + }{ + { + name: "a is greater than b", + a: "v1.17.8", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "b is greater than a", + a: "v1.18.9", + b: "v1.18.8", + output: "v1.18.9", + expectErr: false, + }, + { + name: "b is greater than a", + a: "v1.18", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "a is equal to b", + a: "v1.18.8", + b: "v1.18.8", + output: "v1.18.8", + expectErr: false, + }, + { + name: "a is greater than b and a is major.minor", + a: "v1.18", + b: "v1.17.8", + output: "v1.18", + expectErr: false, + }, + { + name: "a is greater than b and a is major.minor", + a: "1.18", + b: "1.17.8", + output: "1.18", + expectErr: false, + }, + { + name: "a is invalid", + a: "1.18.", + b: "v1.17.8", + output: "v1.17.8", + expectErr: false, + }, + { + name: "b is invalid", + a: "1.18.1", + b: "v1.17.8.", + output: "1.18.1", + expectErr: false, + }, + { + name: "b is invalid", + a: "9.99.9999", + b: "v1.17.8.", + output: "9.99.9999", + expectErr: false, + }, + { + name: "a & b is invalid", + a: "", + b: "v1.17.8.", + output: "", + expectErr: true, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + g := NewWithT(t) + output, err := GetHigherK8sVersion(c.a, c.b) + g.Expect(output).To(Equal(c.output)) + if c.expectErr { + g.Expect(err).NotTo(BeNil()) + } else { + g.Expect(err).To(BeNil()) + } + }) + } +}