From 847cc804c1748ced16d3370dfbf74c941996edd3 Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Thu, 21 Nov 2024 14:44:20 -0800 Subject: [PATCH] ci: add apiserver-ilb templates Signed-off-by: Jack Francis --- .../cluster-template-apiserver-ilb-README.md | 3 + templates/cluster-template-apiserver-ilb.yaml | 213 ++++++++++++ templates/cluster-template-azure-bastion.yaml | 10 +- templates/cluster-template-azure-cni-v1.yaml | 10 +- templates/cluster-template-edgezone.yaml | 10 +- templates/cluster-template-ephemeral.yaml | 10 +- ...r-template-windows-apiserver-ilb-README.md | 3 + ...luster-template-windows-apiserver-ilb.yaml | 306 ++++++++++++++++++ templates/cluster-template-windows.yaml | 14 +- templates/cluster-template.yaml | 10 +- .../flavors/apiserver-ilb/kustomization.yaml | 15 + .../apiserver-ilb/machine-deployment.yaml | 58 ++++ .../patches/control-plane.yaml | 0 .../patches/kubeadm-config-template.yaml | 0 templates/flavors/default/kustomization.yaml | 2 - .../windows-apiserver-ilb/kustomization.yaml | 13 + .../machine-deployment-windows.yaml | 84 +++++ .../patches/kubeadm-config-template.yaml | 0 templates/flavors/windows/kustomization.yaml | 1 - .../cluster-template-prow-azure-cni-v1.yaml | 10 +- .../ci/cluster-template-prow-custom-vnet.yaml | 10 +- .../ci/cluster-template-prow-edgezone.yaml | 10 +- .../test/ci/cluster-template-prow-spot.yaml | 10 +- 23 files changed, 706 insertions(+), 96 deletions(-) create mode 100644 templates/cluster-template-apiserver-ilb-README.md create mode 100644 templates/cluster-template-apiserver-ilb.yaml create mode 100644 templates/cluster-template-windows-apiserver-ilb-README.md create mode 100644 templates/cluster-template-windows-apiserver-ilb.yaml create mode 100644 templates/flavors/apiserver-ilb/kustomization.yaml create mode 100644 templates/flavors/apiserver-ilb/machine-deployment.yaml rename templates/flavors/{default => apiserver-ilb}/patches/control-plane.yaml (100%) rename templates/flavors/{default => apiserver-ilb}/patches/kubeadm-config-template.yaml (100%) create mode 100644 templates/flavors/windows-apiserver-ilb/kustomization.yaml create mode 100644 templates/flavors/windows-apiserver-ilb/machine-deployment-windows.yaml rename templates/flavors/{windows => windows-apiserver-ilb}/patches/kubeadm-config-template.yaml (100%) diff --git a/templates/cluster-template-apiserver-ilb-README.md b/templates/cluster-template-apiserver-ilb-README.md new file mode 100644 index 00000000000..26203f98471 --- /dev/null +++ b/templates/cluster-template-apiserver-ilb-README.md @@ -0,0 +1,3 @@ +# How To Use + +// TODO add brief notes here about the requirement that this template is designed to be used only when the "apiserver-ilb" feature flag is enabled, and how to enable that feature flag. diff --git a/templates/cluster-template-apiserver-ilb.yaml b/templates/cluster-template-apiserver-ilb.yaml new file mode 100644 index 00000000000..a18b181f968 --- /dev/null +++ b/templates/cluster-template-apiserver-ilb.yaml @@ -0,0 +1,213 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + name: ${CLUSTER_NAME}-api-lb + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index ba29dfeb37a..bbae285f2b6 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -31,12 +31,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -197,9 +191,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml index 7704dcfc083..080ceb6eb53 100644 --- a/templates/cluster-template-azure-cni-v1.yaml +++ b/templates/cluster-template-azure-cni-v1.yaml @@ -29,12 +29,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -204,9 +198,7 @@ spec: cloud-provider: external max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index fe5b94184d0..cd0da359eda 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -32,12 +32,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -198,9 +192,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 8415a8f31b7..8f985d81a5d 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -29,12 +29,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -201,9 +195,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/cluster-template-windows-apiserver-ilb-README.md b/templates/cluster-template-windows-apiserver-ilb-README.md new file mode 100644 index 00000000000..26203f98471 --- /dev/null +++ b/templates/cluster-template-windows-apiserver-ilb-README.md @@ -0,0 +1,3 @@ +# How To Use + +// TODO add brief notes here about the requirement that this template is designed to be used only when the "apiserver-ilb" feature flag is enabled, and how to enable that feature flag. diff --git a/templates/cluster-template-windows-apiserver-ilb.yaml b/templates/cluster-template-windows-apiserver-ilb.yaml new file mode 100644 index 00000000000..ecee4ae0eaf --- /dev/null +++ b/templates/cluster-template-windows-apiserver-ilb.yaml @@ -0,0 +1,306 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: calico + csi-proxy: enabled + windows: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: ${CLUSTER_NAME}-control-plane + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureCluster + name: ${CLUSTER_NAME} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureCluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + networkSpec: + apiServerLB: + frontendIPs: + - name: ${CLUSTER_NAME}-api-lb + publicIP: + dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com + name: ${CLUSTER_NAME}-api-lb + subnets: + - name: control-plane-subnet + role: control-plane + - name: node-subnet + role: node + vnet: + name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} + resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + kubeadmConfigSpec: + clusterConfiguration: + apiServer: + extraArgs: + cloud-provider: external + timeoutForControlPlane: 20m + controllerManager: + extraArgs: + allocate-node-cidrs: "false" + cloud-provider: external + cluster-name: ${CLUSTER_NAME} + etcd: + local: + dataDir: /var/lib/etcddisk/etcd + extraArgs: + quota-backend-bytes: "8589934592" + diskSetup: + filesystems: + - device: /dev/disk/azure/scsi1/lun0 + extraOpts: + - -E + - lazy_itable_init=1,lazy_journal_init=1 + filesystem: ext4 + label: etcd_disk + - device: ephemeral0.1 + filesystem: ext4 + label: ephemeral0 + replaceFS: ntfs + partitions: + - device: /dev/disk/azure/scsi1/lun0 + layout: true + overwrite: false + tableType: gpt + files: + - contentFrom: + secret: + key: control-plane-azure.json + name: ${CLUSTER_NAME}-control-plane-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + mounts: + - - LABEL=etcd_disk + - /var/lib/etcddisk + postKubeadmCommands: [] + preKubeadmCommands: [] + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-control-plane + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + dataDisks: + - diskSizeGB: 256 + lun: 0 + nameSuffix: etcddisk + identity: UserAssigned + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-0 + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-0 + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + osDisk: + diskSizeGB: 128 + osType: Linux + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-0-azure-json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: + - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' + >> /etc/hosts +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY} + tenantID: ${AZURE_TENANT_ID} + type: ${CLUSTER_IDENTITY_TYPE:=WorkloadIdentity} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + clusterName: ${CLUSTER_NAME} + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: null + template: + spec: + bootstrap: + configRef: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-md-win + clusterName: ${CLUSTER_NAME} + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate + name: ${CLUSTER_NAME}-md-win + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + annotations: + runtime: containerd + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + metadata: + annotations: + runtime: containerd + spec: + identity: UserAssigned + osDisk: + diskSizeGB: 128 + managedDisk: + storageAccountType: Premium_LRS + osType: Windows + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} + vmSize: ${AZURE_NODE_MACHINE_TYPE} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-md-win + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: ${CLUSTER_NAME}-md-win-azure-json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe + path: C:/defender-exclude-calico.ps1 + permissions: "0744" + joinConfiguration: + nodeRegistration: + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + v: "2" + windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS + name: '{{ ds.meta_data["local_hostname"] }}' + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + preKubeadmCommands: + - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' + -Value '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + users: + - groups: Administrators + name: capi + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index ecee4ae0eaf..0758087f50c 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -33,12 +33,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -199,9 +193,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity @@ -296,9 +288,7 @@ spec: postKubeadmCommands: - nssm set kubelet start SERVICE_AUTO_START - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' - -Value '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'" + preKubeadmCommands: [] users: - groups: Administrators name: capi diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index a18b181f968..3473fafb3d8 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -29,12 +29,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -195,9 +189,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/flavors/apiserver-ilb/kustomization.yaml b/templates/flavors/apiserver-ilb/kustomization.yaml new file mode 100644 index 00000000000..a530a32c6ab --- /dev/null +++ b/templates/flavors/apiserver-ilb/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../base +- machine-deployment.yaml +- ../../azure-cluster-identity + +patches: +- path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml +- path: patches/control-plane.yaml +- path: patches/kubeadm-config-template.yaml + +sortOptions: + order: fifo diff --git a/templates/flavors/apiserver-ilb/machine-deployment.yaml b/templates/flavors/apiserver-ilb/machine-deployment.yaml new file mode 100644 index 00000000000..9aaf668473f --- /dev/null +++ b/templates/flavors/apiserver-ilb/machine-deployment.yaml @@ -0,0 +1,58 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + vmSize: ${AZURE_NODE_MACHINE_TYPE} + osDisk: + osType: "Linux" + diskSizeGB: 128 + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + preKubeadmCommands: [] + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data["local_hostname"] }}' + kubeletExtraArgs: + cloud-provider: external + files: + - contentFrom: + secret: + name: ${CLUSTER_NAME}-md-0-azure-json + key: worker-node-azure.json + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" diff --git a/templates/flavors/default/patches/control-plane.yaml b/templates/flavors/apiserver-ilb/patches/control-plane.yaml similarity index 100% rename from templates/flavors/default/patches/control-plane.yaml rename to templates/flavors/apiserver-ilb/patches/control-plane.yaml diff --git a/templates/flavors/default/patches/kubeadm-config-template.yaml b/templates/flavors/apiserver-ilb/patches/kubeadm-config-template.yaml similarity index 100% rename from templates/flavors/default/patches/kubeadm-config-template.yaml rename to templates/flavors/apiserver-ilb/patches/kubeadm-config-template.yaml diff --git a/templates/flavors/default/kustomization.yaml b/templates/flavors/default/kustomization.yaml index a530a32c6ab..3be9b73f0b2 100644 --- a/templates/flavors/default/kustomization.yaml +++ b/templates/flavors/default/kustomization.yaml @@ -8,8 +8,6 @@ resources: patches: - path: ../../azure-cluster-identity/azurecluster-identity-ref.yaml -- path: patches/control-plane.yaml -- path: patches/kubeadm-config-template.yaml sortOptions: order: fifo diff --git a/templates/flavors/windows-apiserver-ilb/kustomization.yaml b/templates/flavors/windows-apiserver-ilb/kustomization.yaml new file mode 100644 index 00000000000..f75b802a39c --- /dev/null +++ b/templates/flavors/windows-apiserver-ilb/kustomization.yaml @@ -0,0 +1,13 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: +- ../apiserver-ilb +- machine-deployment-windows.yaml + +patches: +- path: ../base-windows-containerd/cluster.yaml +- path: patches/kubeadm-config-template.yaml + +sortOptions: + order: fifo diff --git a/templates/flavors/windows-apiserver-ilb/machine-deployment-windows.yaml b/templates/flavors/windows-apiserver-ilb/machine-deployment-windows.yaml new file mode 100644 index 00000000000..d6b734abf9d --- /dev/null +++ b/templates/flavors/windows-apiserver-ilb/machine-deployment-windows.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-win" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-win" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-win" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-win" + annotations: + runtime: containerd +spec: + template: + metadata: + annotations: + runtime: containerd + spec: + vmSize: ${AZURE_NODE_MACHINE_TYPE} + osDisk: + osType: "Windows" + diskSizeGB: 128 + managedDisk: + storageAccountType: "Premium_LRS" + sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} + identity: UserAssigned + userAssignedIdentities: + - providerID: /subscriptions/${AZURE_SUBSCRIPTION_ID}/resourceGroups/${CI_RG:=capz-ci}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/${USER_IDENTITY:=cloud-provider-user-identity} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-win" +spec: + template: + spec: + users: + - name: capi + groups: Administrators + sshAuthorizedKeys: + - ${AZURE_SSH_PUBLIC_KEY:=""} + preKubeadmCommands: [] + postKubeadmCommands: + - nssm set kubelet start SERVICE_AUTO_START + - powershell C:/defender-exclude-calico.ps1 + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data["local_hostname"] }}' + criSocket: npipe:////./pipe/containerd-containerd + kubeletExtraArgs: + cloud-provider: external + v: "2" + windows-priorityclass: "ABOVE_NORMAL_PRIORITY_CLASS" + files: + - contentFrom: + secret: + name: ${CLUSTER_NAME}-md-win-azure-json + key: worker-node-azure.json + owner: root:root + path: c:/k/azure.json + permissions: "0644" + - path: C:/defender-exclude-calico.ps1 + permissions: "0744" + content: |- + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe + Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe diff --git a/templates/flavors/windows/patches/kubeadm-config-template.yaml b/templates/flavors/windows-apiserver-ilb/patches/kubeadm-config-template.yaml similarity index 100% rename from templates/flavors/windows/patches/kubeadm-config-template.yaml rename to templates/flavors/windows-apiserver-ilb/patches/kubeadm-config-template.yaml diff --git a/templates/flavors/windows/kustomization.yaml b/templates/flavors/windows/kustomization.yaml index 452f43e02c5..b7141420442 100644 --- a/templates/flavors/windows/kustomization.yaml +++ b/templates/flavors/windows/kustomization.yaml @@ -7,7 +7,6 @@ resources: patches: - path: ../base-windows-containerd/cluster.yaml -- path: patches/kubeadm-config-template.yaml sortOptions: order: fifo diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 760210ba5a5..103b5b6b8d2 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -35,12 +35,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -211,9 +205,7 @@ spec: cloud-provider: external max-pods: "110" name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index e9b090f4ca5..b450a762643 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -36,12 +36,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: ${AZURE_CUSTOM_VNET_NAME}-controlplane-subnet role: control-plane @@ -215,9 +209,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index ef1b007edf2..9a7bcd814f7 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -39,12 +39,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION_EDGEZONE} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -214,9 +208,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity diff --git a/templates/test/ci/cluster-template-prow-spot.yaml b/templates/test/ci/cluster-template-prow-spot.yaml index 5dc7f601945..c64b3649111 100644 --- a/templates/test/ci/cluster-template-prow-spot.yaml +++ b/templates/test/ci/cluster-template-prow-spot.yaml @@ -36,12 +36,6 @@ spec: name: ${CLUSTER_IDENTITY_NAME} location: ${AZURE_LOCATION} networkSpec: - apiServerLB: - frontendIPs: - - name: ${CLUSTER_NAME}-api-lb - publicIP: - dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com - name: ${CLUSTER_NAME}-api-lb subnets: - name: control-plane-subnet role: control-plane @@ -211,9 +205,7 @@ spec: kubeletExtraArgs: cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: - - echo '10.0.0.100 ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' - >> /etc/hosts + preKubeadmCommands: [] --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: AzureClusterIdentity