Cherrypicked upstream azure env changes for managed cluster aks on gov cloud

api/v1beta1/types_class.go

@@ -16,7 +16,9 @@ limitations under the License.
 
 package v1beta1
 
-import corev1 "k8s.io/api/core/v1"
+import (
+	corev1 "k8s.io/api/core/v1"
+)
 
 // AzureClusterClassSpec defines the AzureCluster properties that may be shared across several Azure clusters.
 type AzureClusterClassSpec struct {

azure/defaults.go

@@ -31,6 +31,12 @@
 	DefaultUserName = "capi"
 	// DefaultAKSUserName is the default username for a created AKS VM.
 	DefaultAKSUserName = "azureuser"
+	// PublicCloudName is the name of the Azure public cloud.
+	PublicCloudName = "AzurePublicCloud"
+	// ChinaCloudName is the name of the Azure China cloud.
+	ChinaCloudName = "AzureChinaCloud"
+	// USGovernmentCloudName is the name of the Azure US Government cloud.
+	USGovernmentCloudName = "AzureUSGovernmentCloud"
 )
 
 const (
@@ -53,7 +59,7 @@
 
 const (
 	// DefaultWindowsOsAndVersion is the default Windows Server version to use when
 	// genearating default images for Windows nodes.
 	DefaultWindowsOsAndVersion = "windows-2019"
 )
 

azure/scope/managedcontrolplane.go

@@ -70,7 +70,7 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane
 	}
 
 	if params.ControlPlane.Spec.IdentityRef == nil {
-		if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, ""); err != nil {
+		if err := params.AzureClients.setCredentials(params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment); err != nil {
 			return nil, errors.Wrap(err, "failed to create Azure session")
 		}
 	} else {
@@ -79,7 +79,7 @@ func NewManagedControlPlaneScope(ctx context.Context, params ManagedControlPlane
 			return nil, errors.Wrap(err, "failed to init credentials provider")
 		}
 
-		if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, "", credentialsProvider); err != nil {
+		if err := params.AzureClients.setCredentialsWithProvider(ctx, params.ControlPlane.Spec.SubscriptionID, params.ControlPlane.Spec.AzureEnvironment, credentialsProvider); err != nil {
 			return nil, errors.Wrap(err, "failed to configure azure settings and credentials for Identity")
 		}
 	}

config/crd/bases/infrastructure.cluster.x-k8s.io_azuremanagedcontrolplanes.yaml

@@ -593,6 +593,12 @@ spec:
                 required:
                 - upgradeChannel
                 type: object
+              azureEnvironment:
+                description: 'AzureEnvironment is the name of the AzureCloud to be
+                  used. The default value that would be used by most users is "AzurePublicCloud",
+                  other values are: - ChinaCloud: "AzureChinaCloud" - PublicCloud:
+                  "AzurePublicCloud" - USGovernmentCloud: "AzureUSGovernmentCloud"'
+                type: string
               controlPlaneEndpoint:
                 description: ControlPlaneEndpoint represents the endpoint used to
                   communicate with the control plane.

config/default/manager_image_patch.yaml

@@ -8,5 +8,5 @@ spec:
     spec:
       containers:
       # Change the value of image field below to your controller image URL
-      - image: gcr.io/spectro-dev-public/ubuntu/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev
+      - image: gcr.io/spectro-dev-public/shubham/release/cluster-api-azure-controller:v1.3.2-spectro-4.0.0-dev
         name: manager

exp/api/v1beta1/azuremanagedcontrolplane_types.go

@@ -168,6 +168,14 @@ type AzureManagedControlPlaneSpec struct {
 	// +optional
 	AutoUpgradeProfile *ManagedClusterAutoUpgradeProfile `json:"autoUpgradeProfile,omitempty"`
 
+	// AzureEnvironment is the name of the AzureCloud to be used.
+	// The default value that would be used by most users is "AzurePublicCloud", other values are:
+	// - ChinaCloud: "AzureChinaCloud"
+	// - PublicCloud: "AzurePublicCloud"
+	// - USGovernmentCloud: "AzureUSGovernmentCloud"
+	// +optional
+	AzureEnvironment string `json:"azureEnvironment,omitempty"`
+
 	// DisableLocalAccounts - If set to true, getting static credential will be disabled for this cluster. Expected to only be used for AAD clusters.
 	// +optional
 	DisableLocalAccounts *bool `json:"disableLocalAccounts,omitempty"`

exp/api/v1beta1/azuremanagedcontrolplane_webhook.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/utils/ptr"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	"sigs.k8s.io/cluster-api-provider-azure/util/versions"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -77,6 +78,9 @@ func (m *AzureManagedControlPlane) Default(_ client.Client) {
 	m.setDefaultVirtualNetwork()
 	m.setDefaultSubnet()
 	m.setDefaultSku()
+	if m.Spec.AzureEnvironment == "" {
+		m.Spec.AzureEnvironment = azure.PublicCloudName
+	}
 }
 
 // +kubebuilder:webhook:verbs=create;update,path=/validate-infrastructure-cluster-x-k8s-io-v1beta1-azuremanagedcontrolplane,mutating=false,failurePolicy=fail,groups=infrastructure.cluster.x-k8s.io,resources=azuremanagedcontrolplanes,versions=v1beta1,name=validation.azuremanagedcontrolplanes.infrastructure.cluster.x-k8s.io,sideEffects=None,admissionReviewVersions=v1;v1beta1
@@ -253,6 +257,25 @@ func (m *AzureManagedControlPlane) ValidateUpdate(oldRaw runtime.Object, client
 		}
 	}
 
+	if old.Spec.AzureEnvironment != "" {
+		// Prevent AzureEnvironment modification if it was already set to some value
+		if m.Spec.AzureEnvironment == "" {
+			// unsetting the field is not allowed
+			allErrs = append(allErrs,
+				field.Invalid(
+					field.NewPath("Spec", "AzureEnvironment"),
+					m.Spec.AzureEnvironment,
+					"field is immutable, unsetting is not allowed"))
+		} else if m.Spec.AzureEnvironment != old.Spec.AzureEnvironment {
+			// changing the field is not allowed
+			allErrs = append(allErrs,
+				field.Invalid(
+					field.NewPath("Spec", "AzureEnvironment"),
+					*m.Spec.LoadBalancerSKU,
+					"field is immutable"))
+		}
+	}
+
 	if m.Spec.DisableLocalAccounts != nil &&
 		m.Spec.AADProfile == nil {
 		allErrs = append(allErrs,

exp/controllers/azuremanagedmachinepool_reconciler.go

@@ -74,22 +74,28 @@ func (a *AgentPoolVMSSNotFoundError) Is(target error) bool {
 
 // newAzureManagedMachinePoolService populates all the services based on input scope.
 func newAzureManagedMachinePoolService(scope *scope.ManagedMachinePoolScope) (*azureManagedMachinePoolService, error) {
-	var authorizer azure.Authorizer = scope
-	if scope.Location() != "" {
-		regionalAuthorizer, err := azure.WithRegionalBaseURI(scope, scope.Location())
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to create a regional authorizer")
-		}
-		authorizer = regionalAuthorizer
+	scaleSetAuthorizer, err := scaleSetAuthorizer(scope)
+	if err != nil {
+		return nil, err
 	}
 
 	return &azureManagedMachinePoolService{
 		scope:         scope,
 		agentPoolsSvc: agentpools.New(scope),
-		scaleSetsSvc:  scalesets.NewClient(authorizer),
+		scaleSetsSvc:  scalesets.NewClient(scaleSetAuthorizer),
 	}, nil
 }
 
+// scaleSetAuthorizer takes a scope and determines if a regional authorizer is needed for scale sets
+// see https://github.com/kubernetes-sigs/cluster-api-provider-azure/pull/1850 for context on region based authorizer.
+func scaleSetAuthorizer(scope *scope.ManagedMachinePoolScope) (azure.Authorizer, error) {
+	if scope.ControlPlane.Spec.AzureEnvironment == azure.PublicCloudName {
+		return azure.WithRegionalBaseURI(scope, scope.Location()) // public cloud supports regional end points
+	}
+
+	return scope, nil
+}
+
 // Reconcile reconciles all the services in a predetermined order.
 func (s *azureManagedMachinePoolService) Reconcile(ctx context.Context) error {
 	ctx, log, done := tele.StartSpanWithLogger(ctx, "controllers.azureManagedMachinePoolService.Reconcile")

main.go

@@ -18,13 +18,13 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"fmt"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 	"time"
-	"crypto/tls"
 
 	// +kubebuilder:scaffold:imports
 	aadpodv1 "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1"
@@ -35,6 +35,7 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	cgrecord "k8s.io/client-go/tools/record"
+	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"
 	infrav1alpha3 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
@@ -60,7 +61,6 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
-	cliflag "k8s.io/component-base/cli/flag"
 )
 
 type TLSOptions struct {
@@ -69,8 +69,8 @@ type TLSOptions struct {
 }
 
 var (
-	scheme   = runtime.NewScheme()
-	setupLog = ctrl.Log.WithName("setup")
+	scheme     = runtime.NewScheme()
+	setupLog   = ctrl.Log.WithName("setup")
 	tlsOptions = TLSOptions{}
 )
 
@@ -277,10 +277,10 @@ func main() {
 	})
 
 	tlsOptionOverrides, err := GetTLSOptionOverrideFuncs(tlsOptions)
- 	if err != nil {
- 		setupLog.Error(err, "unable to add TLS settings to the webhook server")
- 		os.Exit(1)
- 	}
+	if err != nil {
+		setupLog.Error(err, "unable to add TLS settings to the webhook server")
+		os.Exit(1)
+	}
 
 	restConfig := ctrl.GetConfigOrDie()
 	restConfig.UserAgent = "cluster-api-provider-azure-manager"
@@ -299,7 +299,7 @@ func main() {
 		HealthProbeBindAddress:     healthAddr,
 		Port:                       webhookPort,
 		EventBroadcaster:           broadcaster,
-		TLSOpts: 					tlsOptionOverrides,
+		TLSOpts:                    tlsOptionOverrides,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")