diff --git a/config/capz/manager_image_patch.yaml b/config/capz/manager_image_patch.yaml index 0876a1db40d..354746ca149 100644 --- a/config/capz/manager_image_patch.yaml +++ b/config/capz/manager_image_patch.yaml @@ -8,5 +8,5 @@ spec: spec: containers: # Change the value of image field below to your controller image URL - - image: gcr.io/k8s-staging-cluster-api-azure/cluster-api-azure-controller:latest + - image: localhost:5000/ci-e2e/cluster-api-azure-controller-amd64:20230926200024 name: manager diff --git a/config/capz/manager_pull_policy.yaml b/config/capz/manager_pull_policy.yaml index 74a0879c604..cd7ae12c01e 100644 --- a/config/capz/manager_pull_policy.yaml +++ b/config/capz/manager_pull_policy.yaml @@ -8,4 +8,4 @@ spec: spec: containers: - name: manager - imagePullPolicy: Always + imagePullPolicy: IfNotPresent diff --git a/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml b/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml new file mode 100644 index 00000000000..91a44afeb6d --- /dev/null +++ b/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml @@ -0,0 +1,250 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: default + namespace: default +spec: + controlPlane: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedControlPlaneTemplate + name: ${CLUSTER_NAME}-control-plane + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedClusterTemplate + name: ${CLUSTER_NAME} + workers: + machinePools: + - class: default-system + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool0 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool0 + - class: default-worker + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: ${CLUSTER_NAME}-pool1 + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureManagedMachinePoolTemplate + name: ${CLUSTER_NAME}-pool1 +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + buildProvenance: ${BUILD_PROVENANCE} + creationTimestamp: ${TIMESTAMP} + jobName: ${JOB_NAME} + addonProfiles: + - enabled: true + name: azurepolicy + identityRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: AzureClusterIdentity + name: ${CLUSTER_IDENTITY_NAME} + location: ${AZURE_LOCATION} + subscriptionID: ${AZURE_SUBSCRIPTION_ID} + version: ${KUBERNETES_VERSION} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedClusterTemplate +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + template: + spec: {} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + availabilityZones: + - "1" + - "2" + enableNodePublicIP: false + enableUltraSSD: true + maxPods: 30 + mode: System + name: pool0 + osDiskSizeGB: 30 + osDiskType: Managed + sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + enableNodePublicIP: false + kubeletConfig: + allowedUnsafeSysctls: + - net.* + - kernel.msg* + containerLogMaxFiles: 50 + containerLogMaxSizeMB: 500 + cpuCfsQuota: true + cpuCfsQuotaPeriod: 110ms + cpuManagerPolicy: static + failSwapOn: false + imageGcHighThreshold: 70 + imageGcLowThreshold: 50 + podMaxPids: 2048 + topologyManagerPolicy: best-effort + linuxOSConfig: + swapFileSizeMB: 1500 + sysctls: + fsAioMaxNr: 65536 + fsFileMax: 709620 + fsInotifyMaxUserWatches: 1048576 + fsNrOpen: 1048576 + kernelThreadsMax: 55601 + netCoreNetdevMaxBacklog: 1000 + netCoreOptmemMax: 20480 + netCoreRmemDefault: 212992 + netCoreRmemMax: 212992 + netCoreSomaxconn: 16384 + netCoreWmemDefault: 212992 + netCoreWmemMax: 212992 + netIpv4IPLocalPortRange: 32768 60999 + netIpv4NeighDefaultGcThresh1: 4096 + netIpv4NeighDefaultGcThresh2: 8192 + netIpv4NeighDefaultGcThresh3: 16384 + netIpv4TCPFinTimeout: 60 + netIpv4TCPKeepaliveProbes: 9 + netIpv4TCPKeepaliveTime: 7200 + netIpv4TCPMaxSynBacklog: 16384 + netIpv4TCPMaxTwBuckets: 32768 + netIpv4TCPTwReuse: false + netIpv4TCPkeepaliveIntvl: 75 + netNetfilterNfConntrackBuckets: 65536 + netNetfilterNfConntrackMax: 131072 + vmMaxMapCount: 65530 + vmSwappiness: 60 + vmVfsCachePressure: 100 + transparentHugePageDefrag: madvise + transparentHugePageEnabled: always + maxPods: 64 + mode: User + name: pool1 + nodeLabels: + type: shared + osDiskSizeGB: 40 + osDiskType: Ephemeral + scaleSetPriority: Regular + sku: ${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3} + taints: + - effect: NoSchedule + key: type + value: shared +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureClusterIdentity +metadata: + labels: + clusterctl.cluster.x-k8s.io/move-hierarchy: "true" + name: ${CLUSTER_IDENTITY_NAME} + namespace: default +spec: + allowedNamespaces: {} + clientID: ${AZURE_CLIENT_ID} + clientSecret: + name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} + namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} + tenantID: ${AZURE_TENANT_ID} + type: ServicePrincipal +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool0 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: ${CLUSTER_NAME}-pool1 + namespace: default +spec: + template: + spec: + files: + - contentFrom: + secret: + key: worker-node-azure.json + name: replace_me + owner: root:root + path: /etc/kubernetes/azure.json + permissions: "0644" + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + azure-container-registry-config: /etc/kubernetes/azure.json + cloud-provider: external + name: '{{ ds.meta_data["local_hostname"] }}' + preKubeadmCommands: [] +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: default + version: ${KUBERNETES_VERSION} + workers: + machinePools: + - class: default-system + name: mp-0 + replicas: 1 + - class: default-worker + name: mp-1 + replicas: 1 diff --git a/templates/test/ci/cluster-template-prow-topology.yaml b/templates/test/ci/cluster-template-prow-topology.yaml index e69de29bb2d..2236c297053 100644 --- a/templates/test/ci/cluster-template-prow-topology.yaml +++ b/templates/test/ci/cluster-template-prow-topology.yaml @@ -0,0 +1,304 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + labels: + cni-windows: ${CLUSTER_NAME}-calico + containerd-logger: enabled + csi-proxy: enabled + name: ${CLUSTER_NAME} + namespace: default +spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + topology: + class: ${CLUSTER_CLASS_NAME} + controlPlane: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + variables: + - name: subscriptionID + value: ${AZURE_SUBSCRIPTION_ID} + - name: controlPlaneMachineType + value: ${AZURE_CONTROL_PLANE_MACHINE_TYPE:-""} + - name: workerMachineType + value: ${AZURE_NODE_MACHINE_TYPE:-""} + - name: workerMachineTypeWin + value: ${AZURE_NODE_MACHINE_TYPE:-""} + - name: sshPublicKey + value: ${AZURE_SSH_PUBLIC_KEY_B64:-""} + - name: buildProvenance + value: ${BUILD_PROVENANCE:-""} + - name: timestamp + value: ${TIMESTAMP:-""} + - name: jobName + value: ${JOB_NAME:-""} + - name: clusterIdentityRef + value: ${CLUSTER_IDENTITY_NAME} + - name: location + value: ${AZURE_LOCATION} + - name: k8sFeatureGates + value: ${K8S_FEATURE_GATES:-""} + - name: logLevel + value: "4" + version: ${KUBERNETES_VERSION} + workers: + machineDeployments: + - class: ${CLUSTER_NAME}-worker + name: md-0 + replicas: ${WORKER_MACHINE_COUNT} + - class: ${CLUSTER_NAME}-worker-win + name: md-win + replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: ${CLUSTER_NAME}-calico + namespace: default +spec: + clusterSelector: + matchLabels: + cni-windows: ${CLUSTER_NAME}-calico + resources: + - kind: ConfigMap + name: cni-${CLUSTER_NAME}-calico-windows + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: csi-proxy + namespace: default +spec: + clusterSelector: + matchLabels: + csi-proxy: enabled + resources: + - kind: ConfigMap + name: csi-proxy-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy-windows + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: kube-proxy-windows + template: + metadata: + labels: + k8s-app: kube-proxy-windows + spec: + serviceAccountName: kube-proxy + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\system" + hostNetwork: true + containers: + - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess + args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] + workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" + name: kube-proxy + env: + - name: NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBEPROXY_PATH + valueFrom: + configMapKeyRef: + name: windows-kubeproxy-ci + key: KUBEPROXY_PATH + optional: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + nodeSelector: + kubernetes.io/os: windows + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + volumes: + - configMap: + name: kube-proxy + name: kube-proxy + updateStrategy: + type: RollingUpdate + windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: + IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: + true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n + \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n + \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": + \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": + \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n + \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n + \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": + \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: + v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n + \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: + |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": + [\n {\n \"windows_use_single_network\": true,\n \"type\": + \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n + \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n + \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n + \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": + {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n + \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": + \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": + \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n + \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n + \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": + \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": + \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": + \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n + \ },\n {\n \"Name\": \"EndpointPolicy\",\n + \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": + \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n + \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: + DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n + \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n + \ app: calico\n template:\n metadata:\n labels:\n tier: node\n + \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n + \ nodeSelectorTerms:\n - matchExpressions:\n - + key: kubernetes.io/os\n operator: In\n values:\n + \ - windows\n - key: kubernetes.io/arch\n + \ operator: In\n values:\n - + amd64\n securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: + true\n serviceAccountName: calico-node\n tolerations:\n - operator: + Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on + for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n + \ - effect: NoExecute\n operator: Exists\n initContainers:\n # + This container installs the CNI binaries\n # and CNI network config file + on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n + \ imagePullPolicy: Always\n env:\n # Name of the CNI + config file to create.\n - name: CNI_CONF_NAME\n value: + \"10-calico.conflist\"\n # The CNI network config to install on each + node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n + \ name: calico-config-windows\n key: cni_network_config\n + \ # Set the hostname based on the k8s node name.\n - name: + KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: + spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n + \ valueFrom:\n configMapKeyRef:\n name: + calico-config-windows\n key: veth_mtu\n # Prevents + the container from sleeping forever.\n - name: SLEEP\n value: + \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n + \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: + cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: + cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n + \ securityContext:\n windowsOptions:\n hostProcess: + true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n + \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n + \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n + \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: + Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: + /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n + \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n + \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n + \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: + [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: + Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n + \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n + \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n + \ subPath: static-rules.json\n env:\n - name: POD_NAME\n + \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: + metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: metadata.namespace\n - + name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: + \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n + \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n + \ configMap:\n name: calico-static-rules\n # Used to install + CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n + \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n + \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: + apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n + \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: + IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: + false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n + \ properties:\n apiVersion:\n description: 'APIVersion + defines the versioned schema of this representation\n of an object. + Servers should convert recognized schemas to the latest\n internal + value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n + \ type: string\n kind:\n description: 'Kind is a + string value representing the REST resource this\n object represents. + Servers may infer this from the endpoint the client\n submits requests + to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n + \ type: string\n metadata:\n type: object\n spec:\n + \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n + \ resource.\n properties:\n autoAllocateBlocks:\n + \ type: boolean\n maxBlocksPerHost:\n description: + MaxBlocksPerHost, if non-zero, is the max number of blocks\n that + can be affine to each host.\n maximum: 2147483647\n minimum: + 0\n type: integer\n strictAffinity:\n type: + boolean\n required:\n - autoAllocateBlocks\n - + strictAffinity\n type: object\n type: object\n served: true\n + \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n + \ conditions: []\n storedVersions: []\n" +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cni-${CLUSTER_NAME}-calico-windows + namespace: default +--- +apiVersion: v1 +data: + csi-proxy: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + k8s-app: csi-proxy + name: csi-proxy + namespace: kube-system + spec: + selector: + matchLabels: + k8s-app: csi-proxy + template: + metadata: + labels: + k8s-app: csi-proxy + spec: + nodeSelector: + "kubernetes.io/os": windows + securityContext: + windowsOptions: + hostProcess: true + runAsUserName: "NT AUTHORITY\\SYSTEM" + hostNetwork: true + containers: + - name: csi-proxy + image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: csi-proxy-addon + namespace: default diff --git a/templates/test/ci/prow-aks-clusterclass/kustomization.yaml b/templates/test/ci/prow-aks-clusterclass/kustomization.yaml new file mode 100644 index 00000000000..90ab7444b50 --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: default +resources: + - ../../../flavors/aks-clusterclass + - ../../../flavors/aks-clusterclass-cluster +patchesStrategicMerge: + - patches/tags-aks-clusterclass.yaml + - patches/aks-clusterclass-pool0.yaml + - patches/aks-clusterclass-pool1.yaml + - patches/addons.yaml \ No newline at end of file diff --git a/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml b/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml new file mode 100644 index 00000000000..51d2e93ca8a --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/addons.yaml @@ -0,0 +1,11 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + addonProfiles: + - enabled: true + name: azurepolicy diff --git a/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml new file mode 100644 index 00000000000..0c7b26b42bf --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool0.yaml @@ -0,0 +1,15 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: "${CLUSTER_NAME}-pool0" +spec: + template: + spec: + maxPods: 30 + osDiskType: "Managed" + osDiskSizeGB: 30 + enableNodePublicIP: false + enableUltraSSD: true + availabilityZones: ["1", "2"] + name: pool0 + sku: "${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}" diff --git a/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml new file mode 100644 index 00000000000..f21c89c324a --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/aks-clusterclass-pool1.yaml @@ -0,0 +1,68 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedMachinePoolTemplate +metadata: + name: "${CLUSTER_NAME}-pool1" +spec: + template: + spec: + maxPods: 64 + osDiskType: "Ephemeral" + osDiskSizeGB: 40 + enableNodePublicIP: false + scaleSetPriority: Regular + taints: + - effect: NoSchedule + key: type + value: shared + nodeLabels: + "type": "shared" + name: pool1 + sku: "${AZURE_AKS_NODE_MACHINE_TYPE:=Standard_D2s_v3}" + kubeletConfig: + cpuManagerPolicy: "static" + cpuCfsQuota: true + cpuCfsQuotaPeriod: "110ms" + imageGcHighThreshold: 70 + imageGcLowThreshold: 50 + topologyManagerPolicy: "best-effort" + allowedUnsafeSysctls: + - "net.*" + - "kernel.msg*" + failSwapOn: false + containerLogMaxSizeMB: 500 + containerLogMaxFiles: 50 + podMaxPids: 2048 + linuxOSConfig: + swapFileSizeMB: 1500 + sysctls: + fsAioMaxNr: 65536 + fsFileMax: 709620 + fsInotifyMaxUserWatches: 1048576 + fsNrOpen: 1048576 + kernelThreadsMax: 55601 + netCoreNetdevMaxBacklog: 1000 + netCoreOptmemMax: 20480 + netCoreRmemDefault: 212992 + netCoreRmemMax: 212992 + netCoreSomaxconn: 16384 + netCoreWmemDefault: 212992 + netCoreWmemMax: 212992 + netIpv4IPLocalPortRange: "32768 60999" + netIpv4NeighDefaultGcThresh1: 4096 + netIpv4NeighDefaultGcThresh2: 8192 + netIpv4NeighDefaultGcThresh3: 16384 + netIpv4TCPFinTimeout: 60 + netIpv4TCPKeepaliveProbes: 9 + netIpv4TCPKeepaliveTime: 7200 + netIpv4TCPMaxSynBacklog: 16384 + netIpv4TCPMaxTwBuckets: 32768 + netIpv4TCPTwReuse: false + netIpv4TCPkeepaliveIntvl: 75 + netNetfilterNfConntrackBuckets: 65536 + netNetfilterNfConntrackMax: 131072 + vmMaxMapCount: 65530 + vmSwappiness: 60 + vmVfsCachePressure: 100 + transparentHugePageDefrag: "madvise" + transparentHugePageEnabled: "always" +--- \ No newline at end of file diff --git a/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml b/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml new file mode 100644 index 00000000000..9a6f76edb6f --- /dev/null +++ b/templates/test/ci/prow-aks-clusterclass/patches/tags-aks-clusterclass.yaml @@ -0,0 +1,12 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: AzureManagedControlPlaneTemplate +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: default +spec: + template: + spec: + additionalTags: + jobName: ${JOB_NAME} + creationTimestamp: ${TIMESTAMP} + buildProvenance: ${BUILD_PROVENANCE} \ No newline at end of file diff --git a/templates/test/ci/prow-topology/kustomization.yaml b/templates/test/ci/prow-topology/kustomization.yaml index accf70be1b2..c412c508d08 100644 --- a/templates/test/ci/prow-topology/kustomization.yaml +++ b/templates/test/ci/prow-topology/kustomization.yaml @@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: default resources: - - ../../../flavors/clusterclass/cluster.yaml + - ../../../flavors/clusterclass-cluster/cluster.yaml - cni-resource-set.yaml - ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml patchesStrategicMerge: diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 699749effab..c38793f5e88 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -126,6 +126,8 @@ providers: targetName: "cluster-template-workload-identity.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks.yaml" targetName: "cluster-template-aks.yaml" + - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-aks-clusterclass.yaml" + targetName: "cluster-template-aks.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-custom-vnet.yaml" targetName: "cluster-template-custom-vnet.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-dual-stack.yaml"