Create and add InternalILB as a CAPZ feature flag

- update development.md
- update manager patch policy with the feature flags.

Makefile

@@ -387,6 +387,7 @@ create-cluster: ## Create a workload development Kubernetes cluster on Azure in
 	EXP_MACHINE_POOL=true \
 	EXP_EDGEZONE=true \
 	EXP_ASO_API=true \
+	EXP_APISERVER_ILB=false \
 	$(MAKE) create-management-cluster \
 	create-workload-cluster
 
@@ -746,7 +747,7 @@ kind-create: $(KUBECTL) ## Create capz kind cluster if needed.
 aks-create: $(KUBECTL) ## Create aks cluster as mgmt cluster.
 	./scripts/aks-as-mgmt.sh
 
-.PHONY: tilt-up
+.PHONY: tilt-up ## TODO: EXP_APISERVER_ILB should be set to "true" either at the template level or via the env as being done here.
 tilt-up: install-tools ## Start tilt and build kind cluster if needed.
 	@if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
 		export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \

config/manager/manager.yaml

@@ -24,7 +24,7 @@ spec:
             - --leader-elect
             - "--diagnostics-address=${CAPZ_DIAGNOSTICS_ADDRESS:=:8443}"
             - "--insecure-diagnostics=${CAPZ_INSECURE_DIAGNOSTICS:=false}"
-            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}"
+            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}"
             - "--v=0"
           image: controller:latest
           imagePullPolicy: Always

docs/book/src/developers/development.md

@@ -21,6 +21,8 @@
     - [Tilt for dev in CAPZ](#tilt-for-dev-in-capz)
     - [Tilt for dev in both CAPZ and CAPI](#tilt-for-dev-in-both-capz-and-capi)
     - [Deploying a workload cluster](#deploying-a-workload-cluster)
+    - [Tilt for development (Microsoft Tenant)](#tilt-for-development-microsoft-tenant)
+      - [Flavors for development(Microsoft Tenant)](#flavors-for-development-microsoft-tenant)
     - [Viewing Telemetry](#viewing-telemetry)
     - [Debugging](#debugging)
   - [Manual Testing](#manual-testing)
@@ -151,6 +153,11 @@ development will span both CAPZ and CAPI, then follow the [CAPI and CAPZ instruc
 
 #### Tilt for dev in CAPZ
 
+<aside class="note warning">
+<h2> Warning </h2>
+If you are a MS Tenant and want to develop in CAPZ, navigate to [Tilt for development (Microsoft Tenant)](#tilt-for-development-microsoft-tenant).
+</aside>
+
 If you want to develop in CAPZ and get a local development cluster working quickly, this is the path for you.
 
 Create a file named `tilt-settings.yaml` in the root of the CAPZ repository with the following contents:
@@ -247,6 +254,35 @@ make delete-workload-cluster
 
 > Check out the [self-managed](../self-managed/troubleshooting.md) and [managed](../managed/troubleshooting.md) troubleshooting guides for common errors you might run into.
 
+#### Tilt for development (Microsoft Tenant)
+
+The major difference between developing in CAPZ for MS Tenant and developing in CAPZ for non-MS Tenant is the enabling of the `APIServerILB` feature flag.
+Microsoft tenants are required to use the `APIServerILB` feature flag to facilitate intern-node communication in the workload cluster.
+You can set this by exporting `EXP_APISERVER_ILB=true` in your shell, i.e. run `export EXP_APISERVER_ILB=true`.
+We also encourage you to use AKS cluster as your management cluster.
+
+Outline of the steps:
+- `make clean`
+- `make generate`
+- Set `REGISTRY` in your env. Preferably an Azure Container Registry.
+- Run `docker-build-all` to build all the images.
+- Run `make acr-login` to login to your ACR.
+- Run `docker-push-all` to push all the images to your ACR.
+- Run `make aks-create` to create an AKS cluster. _Notice the changes that get applied to the `tilt-settings.yaml` file._
+- Run `make tilt-up` to start Tilt.
+
+```
+TODO:
+1. Come up with a shorter make target to do all the above steps for MS Tenants.
+2. VNet peering should be exported out as a shell script for users to run.
+```
+
+##### Flavors for development (Microsoft Tenant)
+
+There are two flavors available for development in CAPZ for MSFT Tenant:
+- [apiserver-ilb](../../../../templates/cluster-template-apiserver-ilb.yaml): VM based default flavor that brings up native K8s clusters with Linux nodes.
+- [apiserver-ilb-windows](../../../../templates/cluster-template-windows-apiserver-ilb.yaml): VM based flavor that brings up native K8s clusters with Linux and Windows nodes.
+
 #### Viewing Telemetry
 
 The CAPZ controller emits tracing and metrics data. When run in Tilt, the KinD management cluster is

feature/feature.go

@@ -52,6 +52,12 @@ const (
 	// owner: @nojnhuh
 	// alpha: v1.15
 	ASOAPI featuregate.Feature = "ASOAPI"
+
+	// APIServerILB is a CAPZ feature gate to create an internal LB for the API Server of the workload cluster.
+	// Defaults to false.
+	// owner: @nawazkh
+	// alpha: v1.18
+	APIServerILB featuregate.Feature = "APIServerILB"
 )
 
 func init() {
@@ -66,4 +72,5 @@ var defaultCAPZFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	AKSResourceHealth: {Default: false, PreRelease: featuregate.Alpha},
 	EdgeZone:          {Default: false, PreRelease: featuregate.Alpha},
 	ASOAPI:            {Default: true, PreRelease: featuregate.Alpha},
+	APIServerILB:      {Default: false, PreRelease: featuregate.Alpha},
 }

hack/observability/opentelemetry/controller-manager-patch.yaml

@@ -12,5 +12,5 @@ spec:
             - "--diagnostics-address=:8080"
             - "--insecure-diagnostics"
             - "--leader-elect"
-            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true}"
+            - "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},AKSResourceHealth=${EXP_AKS_RESOURCE_HEALTH:=false},EdgeZone=${EXP_EDGEZONE:=false},ASOAPI=${EXP_ASO_API:=true},APIServerILB=${EXP_APISERVER_ILB:=false}"
             - "--enable-tracing"