diff --git a/Tiltfile b/Tiltfile index 766e125a991..1c8813fd5f0 100644 --- a/Tiltfile +++ b/Tiltfile @@ -388,13 +388,11 @@ def get_addons(flavor_name): if "aks" in flavor_name: return "" - addon_cmd = "" - if "intree-cloud-provider" not in flavor_name: - addon_cmd += "; export CIDRS=$(" + kubectl_cmd + " get cluster ${CLUSTER_NAME} -o jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[*]}')" - addon_cmd += "; export CIDR_LIST=$(bash -c 'echo $CIDRS' | tr ' ' ',')" - addon_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set cloudControllerManager.clusterCIDR=${CIDR_LIST}" - if "flatcar" in flavor_name: # append caCetDir location to the cloud-provider-azure helm install command for flatcar flavor - addon_cmd += " --set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates" + addon_cmd = "; export CIDRS=$(" + kubectl_cmd + " get cluster ${CLUSTER_NAME} -o jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[*]}')" + addon_cmd += "; export CIDR_LIST=$(bash -c 'echo $CIDRS' | tr ' ' ',')" + addon_cmd += "; " + helm_cmd + " --kubeconfig ./${CLUSTER_NAME}.kubeconfig install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure --generate-name --set infra.clusterName=${CLUSTER_NAME} --set cloudControllerManager.clusterCIDR=${CIDR_LIST}" + if "flatcar" in flavor_name: # append caCetDir location to the cloud-provider-azure helm install command for flatcar flavor + addon_cmd += " --set-string cloudControllerManager.caCertDir=/usr/share/ca-certificates" if "azure-cni-v1" in flavor_name: addon_cmd += "; " + kubectl_cmd + " apply -f ./templates/addons/azure-cni-v1.yaml --kubeconfig ./${CLUSTER_NAME}.kubeconfig" diff --git a/scripts/ci-entrypoint.sh b/scripts/ci-entrypoint.sh index 9709cf2ef66..9591c717f08 100755 --- a/scripts/ci-entrypoint.sh +++ b/scripts/ci-entrypoint.sh @@ -58,12 +58,17 @@ setup() { echo "Will use the ${IMAGE_REGISTRY}/${CCM_IMAGE_NAME}:${IMAGE_TAG_CCM} cloud-controller-manager image for external cloud-provider-cluster" echo "Will use the ${IMAGE_REGISTRY}/${CNM_IMAGE_NAME}:${IMAGE_TAG_CNM} cloud-node-manager image for external cloud-provider-azure cluster" - CCM_IMG_ARGS=(--set cloudControllerManager.imageRepository="${IMAGE_REGISTRY}" - --set cloudNodeManager.imageRepository="${IMAGE_REGISTRY}" - --set cloudControllerManager.imageName="${CCM_IMAGE_NAME}" - --set cloudNodeManager.imageName="${CNM_IMAGE_NAME}" - --set-string cloudControllerManager.imageTag="${IMAGE_TAG_CCM}" - --set-string cloudNodeManager.imageTag="${IMAGE_TAG_CNM}") + if [[ -n "${LOAD_CLOUD_CONFIG_FROM_SECRET:-}" ]]; then + export CLOUD_CONFIG="" + export CONFIG_SECRET_NAME="azure-cloud-provider" + export ENABLE_DYNAMIC_RELOADING=true + until copy_secret; do + sleep 5 + done + fi + + export CCM_LOG_VERBOSITY="${CCM_LOG_VERBOSITY:-4}" + export CLOUD_PROVIDER_AZURE_LABEL="azure-ci" fi if [[ "$(capz::util::should_build_kubernetes)" == "true" ]]; then @@ -142,33 +147,6 @@ create_cluster() { export KUBE_SSH_USER } -# get_cidrs derives the CIDR from the Cluster's '.spec.clusterNetwork.pods.cidrBlocks' metadata -# any retryable operation in this function must return a non-zero exit code on failure so that we can -# retry it using a `until get_cidrs; do sleep 5; done` pattern; -# and any statement must be idempotent so that subsequent retry attempts can make forward progress. -get_cidrs() { - # Get cluster CIDRs from Cluster object - CIDR0=$(${KUBECTL} --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" get cluster "${CLUSTER_NAME}" -o=jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[0]}') || return 1 - export CIDR0 - CIDR_LENGTH=$(${KUBECTL} --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" get cluster "${CLUSTER_NAME}" -o=jsonpath='{.spec.clusterNetwork.pods.cidrBlocks}' | jq '. | length') || return 1 - if [[ "${CIDR_LENGTH}" == "2" ]]; then - CIDR1=$(${KUBECTL} get cluster --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" "${CLUSTER_NAME}" -o=jsonpath='{.spec.clusterNetwork.pods.cidrBlocks[1]}') || return 1 - export CIDR1 - fi -} - -# get_cloud_provider determines if the Cluster is using an intree or external cloud-provider from the KubeadmConfigSpec. -# any retryable operation in this function must return a non-zero exit code on failure so that we can -# retry it using a `until get_cloud_provider; do sleep 5; done` pattern; -# and any statement must be idempotent so that subsequent retry attempts can make forward progress. -get_cloud_provider() { - CLOUD_PROVIDER=$("${KUBECTL}" --kubeconfig "${REPO_ROOT}/${KIND_CLUSTER_NAME}.kubeconfig" get kubeadmcontrolplane -l cluster.x-k8s.io/cluster-name="${CLUSTER_NAME}" -o=jsonpath='{.items[0].spec.kubeadmConfigSpec.clusterConfiguration.controllerManager.extraArgs.cloud-provider}') || return 1 - if [[ "${CLOUD_PROVIDER:-}" = "azure" ]]; then - IN_TREE="true" - export IN_TREE - fi -} - # copy_kubeadm_config_map copies the kubeadm configmap into the calico-system namespace. # any retryable operation in this function must return a non-zero exit code on failure so that we can # retry it using a `until copy_kubeadm_config_map; do sleep 5; done` pattern; @@ -185,39 +163,6 @@ copy_kubeadm_config_map() { fi } -# install_cloud_provider_azure installs OOT cloud-provider-azure componentry onto the Cluster. -# Any retryable operation in this function must return a non-zero exit code on failure so that we can -# retry it using a `until install_cloud_provider_azure; do sleep 5; done` pattern; -# and any statement must be idempotent so that subsequent retry attempts can make forward progress. -install_cloud_provider_azure() { - CLOUD_CONFIG="/etc/kubernetes/azure.json" - CONFIG_SECRET_NAME="" - ENABLE_DYNAMIC_RELOADING=false - if [[ -n "${LOAD_CLOUD_CONFIG_FROM_SECRET:-}" ]]; then - CLOUD_CONFIG="" - CONFIG_SECRET_NAME="azure-cloud-provider" - ENABLE_DYNAMIC_RELOADING=true - copy_secret || return 1 - fi - - CCM_CLUSTER_CIDR="${CIDR0}" - if [[ -n "${CIDR1:-}" ]]; then - CCM_CLUSTER_CIDR="${CIDR0}\,${CIDR1}" - fi - echo "CCM cluster CIDR: ${CCM_CLUSTER_CIDR:-}" - - export CCM_LOG_VERBOSITY="${CCM_LOG_VERBOSITY:-4}" - echo "Installing cloud-provider-azure components via helm" - "${HELM}" upgrade cloud-provider-azure --install --repo https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo cloud-provider-azure \ - --set infra.clusterName="${CLUSTER_NAME}" \ - --set cloudControllerManager.replicas="${CCM_COUNT}" \ - --set cloudControllerManager.enableDynamicReloading="${ENABLE_DYNAMIC_RELOADING}" \ - --set cloudControllerManager.cloudConfig="${CLOUD_CONFIG}" \ - --set cloudControllerManager.cloudConfigSecretName="${CONFIG_SECRET_NAME}" \ - --set cloudControllerManager.logVerbosity="${CCM_LOG_VERBOSITY}" \ - --set-string cloudControllerManager.clusterCIDR="${CCM_CLUSTER_CIDR}" "${CCM_IMG_ARGS[@]}" || return 1 -} - # wait_for_nodes returns when all nodes in the workload cluster are Ready. wait_for_nodes() { echo "Waiting for ${CONTROL_PLANE_MACHINE_COUNT} control plane machine(s), ${WORKER_MACHINE_COUNT} worker machine(s), and ${WINDOWS_WORKER_MACHINE_COUNT:-0} windows machine(s) to become Ready" @@ -255,23 +200,11 @@ wait_for_pods() { } install_addons() { - until get_cidrs; do - sleep 5 - done # export the target cluster KUBECONFIG if not already set export KUBECONFIG="${KUBECONFIG:-${PWD}/kubeconfig}" until copy_kubeadm_config_map; do sleep 5 done - # install cloud-provider-azure components, if using out-of-tree - until get_cloud_provider; do - sleep 5 - done - if [[ -z "${IN_TREE:-}" ]]; then - until install_cloud_provider_azure; do - sleep 5 - done - fi # In order to determine the successful outcome of CNI and cloud-provider-azure, # we need to wait a little bit for nodes and pods terminal state, # so we block successful return upon the cluster being fully operational. diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml new file mode 100644 index 00000000000..ba05ea4b2f7 --- /dev/null +++ b/templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml @@ -0,0 +1,28 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + clusterSelector: + matchLabels: + cloud-provider: "azure-ci" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure-oot + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} \ No newline at end of file diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure-flatcar.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure-flatcar.yaml new file mode 100644 index 00000000000..c7b95708ab4 --- /dev/null +++ b/templates/addons/cluster-api-helm/cloud-provider-azure-flatcar.yaml @@ -0,0 +1,18 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-flatcar +spec: + clusterSelector: + matchLabels: + cloud-provider: "azure-flatcar" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure-oot + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 + caCertDir: /usr/share/ca-certificates \ No newline at end of file diff --git a/templates/addons/cluster-api-helm/cloud-provider-azure.yaml b/templates/addons/cluster-api-helm/cloud-provider-azure.yaml new file mode 100644 index 00000000000..b56f45ec56c --- /dev/null +++ b/templates/addons/cluster-api-helm/cloud-provider-azure.yaml @@ -0,0 +1,17 @@ +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + clusterSelector: + matchLabels: + cloud-provider: "azure" + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + chartName: cloud-provider-azure + releaseName: cloud-provider-azure-oot + valuesTemplate: | + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 \ No newline at end of file diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index 401e35b2e63..6b8b168c165 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -1,6 +1,8 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} name: ${CLUSTER_NAME} namespace: default spec: @@ -239,3 +241,52 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} diff --git a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml index 7705ce1ce4c..19fec4132cc 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-dual-stack.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico-dual-stack cni-windows: ${CLUSTER_NAME}-calico containerd-logger: disabled @@ -504,6 +505,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: csi-proxy: | diff --git a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml index 0bddbbc10b8..ff362f95dd8 100644 --- a/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico-ipv6 cni-windows: ${CLUSTER_NAME}-calico containerd-logger: disabled @@ -522,6 +523,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: csi-proxy: | diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 898bc691559..27ec11a3080 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -702,6 +703,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 60bca83b607..bc3f3e280bd 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default @@ -296,3 +297,52 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} diff --git a/templates/test/ci/cluster-template-prow-dual-stack.yaml b/templates/test/ci/cluster-template-prow-dual-stack.yaml index 947324021d7..78666cbb708 100644 --- a/templates/test/ci/cluster-template-prow-dual-stack.yaml +++ b/templates/test/ci/cluster-template-prow-dual-stack.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico-dual-stack name: ${CLUSTER_NAME} namespace: default @@ -371,3 +372,52 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index fe59ee7d2a6..8e0ee44e0ad 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default @@ -280,3 +281,52 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} diff --git a/templates/test/ci/cluster-template-prow-flatcar.yaml b/templates/test/ci/cluster-template-prow-flatcar.yaml index 9680d902603..1c6c431744c 100644 --- a/templates/test/ci/cluster-template-prow-flatcar.yaml +++ b/templates/test/ci/cluster-template-prow-flatcar.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: azure-flatcar cni: calico name: ${CLUSTER_NAME} namespace: default @@ -307,3 +308,23 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-flatcar + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-flatcar + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 + caCertDir: /usr/share/ca-certificates diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml deleted file mode 100644 index adfb8ef0e00..00000000000 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider-machine-pool.yaml +++ /dev/null @@ -1,763 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - windows: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - location: ${AZURE_LOCATION} - strategy: - rollingUpdate: - deletePolicy: Oldest - maxSurge: 25% - maxUnavailable: 1 - type: RollingUpdate - template: - osDisk: - diskSizeGB: 30 - managedDisk: - storageAccountType: Premium_LRS - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmExtensions: - - name: CustomScript - protectedSettings: - commandToExecute: | - #!/bin/sh - echo "This script is a no-op used for extension testing purposes ..." - touch test_file - publisher: Microsoft.Azure.Extensions - version: "2.1" - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachinePool -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachinePool - name: ${CLUSTER_NAME}-mp-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachinePool -metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - location: ${AZURE_LOCATION} - template: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-mp-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell c:/create-external-network.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: calico - namespace: default -spec: - chartName: tigera-operator - clusterSelector: - matchLabels: - cni: calico - namespace: tigera-operator - releaseName: projectcalico - repoURL: https://docs.tigera.io/calico/charts - valuesTemplate: |- - installation: - cni: - type: Calico - calicoNetwork: - bgp: Disabled - mtu: 1350 - ipPools: - ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} - - cidr: {{ $cidr }} - encapsulation: VXLAN{{end}} - registry: mcr.microsoft.com/oss - # Image and registry configuration for the tigera/operator pod. - tigeraOperator: - image: tigera/operator - registry: mcr.microsoft.com/oss - calicoctl: - image: mcr.microsoft.com/oss/calico/ctl - version: ${CALICO_VERSION} ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: azuredisk-csi-driver-chart - namespace: default -spec: - chartName: azuredisk-csi-driver - clusterSelector: - matchLabels: - azuredisk-csi: "true" - namespace: kube-system - releaseName: azuredisk-csi-driver-oot - repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts - valuesTemplate: |- - controller: - replicas: 1 - runOnControlPlane: true - windows: - useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} ---- -apiVersion: v1 -data: - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: - v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n - \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: - |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": - [\n {\n \"windows_use_single_network\": true,\n \"type\": - \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n - \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n - \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n - \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": - {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n - \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": - \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": - \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n - \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n - \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": - \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": - \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": - \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n - \ },\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": - \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n - \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: - DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n - \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n - \ app: calico\n template:\n metadata:\n labels:\n tier: node\n - \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - \ nodeSelectorTerms:\n - matchExpressions:\n - - key: kubernetes.io/os\n operator: In\n values:\n - \ - windows\n - key: kubernetes.io/arch\n - \ operator: In\n values:\n - - amd64\n securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: - true\n serviceAccountName: calico-node\n tolerations:\n - operator: - Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on - for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - \ - effect: NoExecute\n operator: Exists\n initContainers:\n # - This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n - \ imagePullPolicy: Always\n env:\n # Name of the CNI - config file to create.\n - name: CNI_CONF_NAME\n value: - \"10-calico.conflist\"\n # The CNI network config to install on each - node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n - \ name: calico-config-windows\n key: cni_network_config\n - \ # Set the hostname based on the k8s node name.\n - name: - KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: - spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n - \ valueFrom:\n configMapKeyRef:\n name: - calico-config-windows\n key: veth_mtu\n # Prevents - the container from sleeping forever.\n - name: SLEEP\n value: - \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n - \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: - cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: - cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n - \ securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n - \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: - Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: - /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n - \ fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n - \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n - \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: - [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: - Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n - \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n - \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n - \ subPath: static-rules.json\n env:\n - name: POD_NAME\n - \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: - \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n - \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n - \ configMap:\n name: calico-static-rules\n # Used to install - CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n - \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n - \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: - apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n - \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: - IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: - false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n - \ properties:\n apiVersion:\n description: 'APIVersion - defines the versioned schema of this representation\n of an object. - Servers should convert recognized schemas to the latest\n internal - value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n - \ type: string\n kind:\n description: 'Kind is a - string value representing the REST resource this\n object represents. - Servers may infer this from the endpoint the client\n submits requests - to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n - \ type: string\n metadata:\n type: object\n spec:\n - \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n - \ resource.\n properties:\n autoAllocateBlocks:\n - \ type: boolean\n maxBlocksPerHost:\n description: - MaxBlocksPerHost, if non-zero, is the max number of blocks\n that - can be affine to each host.\n maximum: 2147483647\n minimum: - 0\n type: integer\n strictAffinity:\n type: - boolean\n required:\n - autoAllocateBlocks\n - - strictAffinity\n type: object\n type: object\n served: true\n - \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n - \ conditions: []\n storedVersions: []\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index 4ee9806a4a1..e69de29bb2d 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -1,826 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - cni: calico - cni-windows: ${CLUSTER_NAME}-calico - containerd-logger: enabled - csi-proxy: enabled - name: ${CLUSTER_NAME} - namespace: default -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} - namespace: default -spec: - additionalTags: - buildProvenance: ${BUILD_PROVENANCE} - creationTimestamp: ${TIMESTAMP} - jobName: ${JOB_NAME} - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: ${CLUSTER_IDENTITY_NAME} - location: ${AZURE_LOCATION} - networkSpec: - subnets: - - name: control-plane-subnet - role: control-plane - - name: node-subnet - role: node - vnet: - name: ${AZURE_VNET_NAME:=${CLUSTER_NAME}-vnet} - resourceGroup: ${AZURE_RESOURCE_GROUP:=${CLUSTER_NAME}} - subscriptionID: ${AZURE_SUBSCRIPTION_ID} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - feature-gates: ${K8S_FEATURE_GATES:-""} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - timeoutForControlPlane: 20m - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - etcd: - local: - dataDir: /var/lib/etcddisk/etcd - extraArgs: - quota-backend-bytes: "8589934592" - diskSetup: - filesystems: - - device: /dev/disk/azure/scsi1/lun0 - extraOpts: - - -E - - lazy_itable_init=1,lazy_journal_init=1 - filesystem: ext4 - label: etcd_disk - - device: ephemeral0.1 - filesystem: ext4 - label: ephemeral0 - replaceFS: ntfs - partitions: - - device: /dev/disk/azure/scsi1/lun0 - layout: true - overwrite: false - tableType: gpt - files: - - contentFrom: - secret: - key: control-plane-azure.json - name: ${CLUSTER_NAME}-control-plane-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - mounts: - - - LABEL=etcd_disk - - /var/lib/etcddisk - postKubeadmCommands: [] - preKubeadmCommands: [] - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - template: - spec: - dataDisks: - - diskSizeGB: 256 - lun: 0 - nameSuffix: etcddisk - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_CONTROL_PLANE_MACHINE_TYPE} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: {} - template: - metadata: - labels: - nodepool: pool1 - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - osDisk: - diskSizeGB: 128 - osType: Linux - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmExtensions: - - name: CustomScript - protectedSettings: - commandToExecute: | - #!/bin/sh - echo "This script is a no-op used for extension testing purposes ..." - touch test_file - publisher: Microsoft.Azure.Extensions - version: "2.1" - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-0-azure-json - owner: root:root - path: /etc/kubernetes/azure.json - permissions: "0644" - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - name: '{{ ds.meta_data["local_hostname"] }}' - preKubeadmCommands: [] ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WINDOWS_WORKER_MACHINE_COUNT:-0} - selector: {} - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-win - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureMachineTemplate - name: ${CLUSTER_NAME}-md-win - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureMachineTemplate -metadata: - annotations: - runtime: containerd - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - metadata: - annotations: - runtime: containerd - windowsServerVersion: ${WINDOWS_SERVER_VERSION:=""} - spec: - osDisk: - diskSizeGB: 128 - managedDisk: - storageAccountType: Premium_LRS - osType: Windows - sshPublicKey: ${AZURE_SSH_PUBLIC_KEY_B64:=""} - vmSize: ${AZURE_NODE_MACHINE_TYPE} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - files: - - contentFrom: - secret: - key: worker-node-azure.json - name: ${CLUSTER_NAME}-md-win-azure-json - owner: root:root - path: c:/k/azure.json - permissions: "0644" - - content: |- - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico.exe - Add-MpPreference -ExclusionProcess C:/opt/cni/bin/calico-ipam.exe - path: C:/defender-exclude-calico.ps1 - permissions: "0744" - - content: | - # /tmp is assumed created and required for upstream e2e tests to pass - New-Item -ItemType Directory -Force -Path C:\tmp\ - path: C:/create-temp-folder.ps1 - permissions: "0744" - - content: | - $ErrorActionPreference = 'Stop' - - $$CONTAINERD_URL="${WINDOWS_CONTAINERD_URL}" - if($$CONTAINERD_URL -ne ""){ - # Kubelet service depends on contianerd service so make a best effort attempt to stop it - Stop-Service kubelet -Force -ErrorAction SilentlyContinue - Stop-Service containerd -Force - echo "downloading containerd: $$CONTAINERD_URL" - curl.exe --retry 10 --retry-delay 5 -L "$$CONTAINERD_URL" --output "c:/k/containerd.tar.gz" - tar.exe -zxvf c:/k/containerd.tar.gz -C "c:/Program Files/containerd" --strip-components 1 - - Start-Service containerd - } - - containerd.exe --version - containerd-shim-runhcs-v1.exe --version - path: C:/replace-containerd.ps1 - permissions: "0744" - - content: | - mkdir -Force c:/localdumps - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpCount /t REG_DWORD /d 50 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpType /t REG_DWORD /d 2 /f - reg.exe add "HKLM\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /V DumpFolder /t REG_EXPAND_SZ /d "c:/LocalDumps" /f - # Enable sftp so we can copy crash dump files during log collection of stfp - $sshd_config = "$env:ProgramData\ssh\sshd_config" - if (-not (Test-Path $sshd_config)) { mkdir -Force $sshd_config } - Add-Content -Path $sshd_config "Subsystem sftp sftp-server.exe" - sc.exe stop sshd - sc.exe start sshd - path: C:/collect-hns-crashes.ps1 - permissions: "0744" - joinConfiguration: - nodeRegistration: - criSocket: npipe:////./pipe/containerd-containerd - kubeletExtraArgs: - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - feature-gates: ${NODE_FEATURE_GATES:-""} - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - name: '{{ ds.meta_data["local_hostname"] }}' - postKubeadmCommands: - - nssm set kubelet start SERVICE_AUTO_START - - powershell C:/defender-exclude-calico.ps1 - preKubeadmCommands: - - powershell C:/create-temp-folder.ps1 - - powershell C:/replace-containerd.ps1 - - powershell C:/collect-hns-crashes.ps1 - users: - - groups: Administrators - name: capi - sshAuthorizedKeys: - - ${AZURE_SSH_PUBLIC_KEY:=""} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineHealthCheck -metadata: - name: ${CLUSTER_NAME}-mhc-0 - namespace: default -spec: - clusterName: ${CLUSTER_NAME} - maxUnhealthy: 100% - selector: - matchLabels: - nodepool: pool1 - unhealthyConditions: - - status: "True" - timeout: 30s - type: E2ENodeUnhealthy ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-calico-windows - namespace: default -spec: - clusterSelector: - matchLabels: - cni-windows: ${CLUSTER_NAME}-calico - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-calico-windows - strategy: ApplyOnce ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" - name: ${CLUSTER_IDENTITY_NAME} - namespace: default -spec: - allowedNamespaces: {} - clientID: ${AZURE_CLIENT_ID} - clientSecret: - name: ${AZURE_CLUSTER_IDENTITY_SECRET_NAME} - namespace: ${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE} - tenantID: ${AZURE_TENANT_ID} - type: ServicePrincipal ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: csi-proxy - namespace: default -spec: - clusterSelector: - matchLabels: - csi-proxy: enabled - resources: - - kind: ConfigMap - name: csi-proxy-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: containerd-logger-${CLUSTER_NAME} - namespace: default -spec: - clusterSelector: - matchLabels: - containerd-logger: enabled - resources: - - kind: ConfigMap - name: containerd-logger-${CLUSTER_NAME} - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: calico - namespace: default -spec: - chartName: tigera-operator - clusterSelector: - matchLabels: - cni: calico - namespace: tigera-operator - releaseName: projectcalico - repoURL: https://docs.tigera.io/calico/charts - valuesTemplate: |- - installation: - cni: - type: Calico - calicoNetwork: - bgp: Disabled - mtu: 1350 - ipPools: - ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} - - cidr: {{ $cidr }} - encapsulation: VXLAN{{end}} - registry: mcr.microsoft.com/oss - # Image and registry configuration for the tigera/operator pod. - tigeraOperator: - image: tigera/operator - registry: mcr.microsoft.com/oss - calicoctl: - image: mcr.microsoft.com/oss/calico/ctl - version: ${CALICO_VERSION} ---- -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: azuredisk-csi-driver-chart - namespace: default -spec: - chartName: azuredisk-csi-driver - clusterSelector: - matchLabels: - azuredisk-csi: "true" - namespace: kube-system - releaseName: azuredisk-csi-driver-oot - repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts - valuesTemplate: |- - controller: - replicas: 1 - runOnControlPlane: true - windows: - useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} ---- -apiVersion: v1 -data: - proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: kube-proxy - name: kube-proxy-windows - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: kube-proxy-windows - template: - metadata: - labels: - k8s-app: kube-proxy-windows - spec: - serviceAccountName: kube-proxy - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: sigwindowstools/kube-proxy:${KUBERNETES_VERSION/+/_}-calico-hostprocess - args: ["$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/start.ps1"] - workingDir: "$env:CONTAINER_SANDBOX_MOUNT_POINT/kube-proxy/" - name: kube-proxy - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: KUBEPROXY_PATH - valueFrom: - configMapKeyRef: - name: windows-kubeproxy-ci - key: KUBEPROXY_PATH - optional: true - volumeMounts: - - mountPath: /var/lib/kube-proxy - name: kube-proxy - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: kube-proxy - name: kube-proxy - updateStrategy: - type: RollingUpdate - windows-cni: "# strictAffinity required for windows\napiVersion: crd.projectcalico.org/v1\nkind: - IPAMConfig\nmetadata:\n name: default\nspec:\n autoAllocateBlocks: true\n strictAffinity: - true\n---\nkind: ConfigMap\napiVersion: v1\nmetadata:\n name: calico-static-rules\n - \ namespace: calico-system\n labels:\n tier: node\n app: calico\ndata:\n - \ static-rules.json: |\n {\n \"Provider\": \"azure\",\n \"Version\": - \"0.1\",\n \"Rules\": [\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Rule\": {\n \"Id\": \"wireserver\",\n \"Type\": - \"ACL\",\n \"Protocol\": 6,\n \"Action\": \"Block\",\n - \ \"Direction\": \"Out\",\n \"RemoteAddresses\": \"168.63.129.16/32\",\n - \ \"RemotePorts\": \"80\",\n \"Priority\": 200,\n \"RuleType\": - \"Switch\"\n }\n }\n ]\n } \n---\nkind: ConfigMap\napiVersion: - v1\nmetadata:\n name: calico-config-windows\n namespace: calico-system\n labels:\n - \ tier: node\n app: calico\ndata:\n veth_mtu: \"1350\"\n \n cni_network_config: - |\n {\n \"name\": \"Calico\",\n \"cniVersion\": \"0.3.1\",\n \"plugins\": - [\n {\n \"windows_use_single_network\": true,\n \"type\": - \"calico\",\n \"mode\": \"vxlan\",\n \"nodename\": \"__KUBERNETES_NODE_NAME__\",\n - \ \"nodename_file_optional\": true,\n \"log_file_path\": \"c:/cni.log\",\n - \ \"log_level\": \"debug\",\n\n \"vxlan_mac_prefix\": \"0E-2A\",\n - \ \"vxlan_vni\": 4096,\n \"mtu\": __CNI_MTU__,\n \"policy\": - {\n \"type\": \"k8s\"\n },\n\n \"log_level\": \"info\",\n\n - \ \"capabilities\": {\"dns\": true},\n \"DNS\": {\n \"Search\": - \ [\n \"svc.cluster.local\"\n ]\n },\n\n \"datastore_type\": - \"kubernetes\",\n\n \"kubernetes\": {\n \"kubeconfig\": \"__KUBECONFIG_FILEPATH__\"\n - \ },\n\n \"ipam\": {\n \"type\": \"calico-ipam\",\n - \ \"subnet\": \"usePodCidr\"\n },\n\n \"policies\": - \ [\n {\n \"Name\": \"EndpointPolicy\",\n \"Value\": - \ {\n \"Type\": \"OutBoundNAT\",\n \"ExceptionList\": - \ [\n \"__K8S_SERVICE_CIDR__\"\n ]\n }\n - \ },\n {\n \"Name\": \"EndpointPolicy\",\n - \ \"Value\": {\n \"Type\": \"SDNROUTE\",\n \"DestinationPrefix\": - \ \"__K8S_SERVICE_CIDR__\",\n \"NeedEncap\": true\n }\n - \ }\n ]\n }\n ]\n\n }\n---\napiVersion: apps/v1\nkind: - DaemonSet\nmetadata:\n name: calico-node-windows\n labels:\n tier: node\n - \ app: calico\n namespace: calico-system\nspec:\n selector:\n matchLabels:\n - \ app: calico\n template:\n metadata:\n labels:\n tier: node\n - \ app: calico\n spec:\n affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n - \ nodeSelectorTerms:\n - matchExpressions:\n - - key: kubernetes.io/os\n operator: In\n values:\n - \ - windows\n - key: kubernetes.io/arch\n - \ operator: In\n values:\n - - amd64\n securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n hostNetwork: - true\n serviceAccountName: calico-node\n tolerations:\n - operator: - Exists\n effect: NoSchedule\n # Mark the pod as a critical add-on - for rescheduling.\n - key: CriticalAddonsOnly\n operator: Exists\n - \ - effect: NoExecute\n operator: Exists\n initContainers:\n # - This container installs the CNI binaries\n # and CNI network config file - on each node.\n - name: install-cni\n image: sigwindowstools/calico-install:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/install.ps1\"]\n - \ imagePullPolicy: Always\n env:\n # Name of the CNI - config file to create.\n - name: CNI_CONF_NAME\n value: - \"10-calico.conflist\"\n # The CNI network config to install on each - node.\n - name: CNI_NETWORK_CONFIG\n valueFrom:\n configMapKeyRef:\n - \ name: calico-config-windows\n key: cni_network_config\n - \ # Set the hostname based on the k8s node name.\n - name: - KUBERNETES_NODE_NAME\n valueFrom:\n fieldRef:\n fieldPath: - spec.nodeName\n # CNI MTU Config variable\n - name: CNI_MTU\n - \ valueFrom:\n configMapKeyRef:\n name: - calico-config-windows\n key: veth_mtu\n # Prevents - the container from sleeping forever.\n - name: SLEEP\n value: - \"false\"\n - name: K8S_SERVICE_CIDR\n value: \"10.96.0.0/12\"\n - \ volumeMounts:\n - mountPath: /host/opt/cni/bin\n name: - cni-bin-dir\n - mountPath: /host/etc/cni/net.d\n name: - cni-net-dir\n - name: kubeadm-config\n mountPath: /etc/kubeadm-config/\n - \ securityContext:\n windowsOptions:\n hostProcess: - true\n runAsUserName: \"NT AUTHORITY\\\\system\"\n containers:\n - \ - name: calico-node-startup\n image: sigwindowstools/calico-node:v3.26.1-hostprocess\n - \ args: [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/node-service.ps1\"]\n - \ workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n imagePullPolicy: - Always\n volumeMounts:\n - name: calico-config-windows\n mountPath: - /etc/kube-calico-windows/\n env:\n - name: POD_NAME\n valueFrom:\n - \ fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: CNI_IPAM_TYPE\n value: \"calico-ipam\"\n - name: CALICO_NETWORKING_BACKEND\n - \ value: \"vxlan\"\n - name: KUBECONFIG\n value: \"C:/etc/cni/net.d/calico-kubeconfig\"\n - \ - name: VXLAN_VNI\n value: \"4096\"\n - name: calico-node-felix\n - \ image: sigwindowstools/calico-node:v3.26.1-hostprocess\n args: - [\"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/felix-service.ps1\"]\n imagePullPolicy: - Always\n workingDir: \"$env:CONTAINER_SANDBOX_MOUNT_POINT/calico/\"\n volumeMounts:\n - \ - name: calico-config-windows\n mountPath: /etc/kube-calico-windows/\n - \ - name: calico-static-rules\n mountPath: /calico/static-rules.json\n - \ subPath: static-rules.json\n env:\n - name: POD_NAME\n - \ valueFrom:\n fieldRef:\n apiVersion: v1\n fieldPath: - metadata.name\n - name: POD_NAMESPACE\n valueFrom:\n fieldRef:\n - \ apiVersion: v1\n fieldPath: metadata.namespace\n - - name: VXLAN_VNI\n value: \"4096\"\n - name: KUBECONFIG\n value: - \"C:/etc/cni/net.d/calico-kubeconfig\"\n volumes:\n - name: calico-config-windows\n - \ configMap:\n name: calico-config-windows\n - name: calico-static-rules\n - \ configMap:\n name: calico-static-rules\n # Used to install - CNI.\n - name: cni-bin-dir\n hostPath:\n path: /opt/cni/bin\n - \ - name: cni-net-dir\n hostPath:\n path: /etc/cni/net.d\n - \ - name: kubeadm-config\n configMap:\n name: kubeadm-config\n---\napiVersion: - apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: ipamconfigs.crd.projectcalico.org\nspec:\n - \ group: crd.projectcalico.org\n names:\n kind: IPAMConfig\n listKind: - IPAMConfigList\n plural: ipamconfigs\n singular: ipamconfig\n preserveUnknownFields: - false\n scope: Cluster\n versions:\n - name: v1\n schema:\n openAPIV3Schema:\n - \ properties:\n apiVersion:\n description: 'APIVersion - defines the versioned schema of this representation\n of an object. - Servers should convert recognized schemas to the latest\n internal - value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'\n - \ type: string\n kind:\n description: 'Kind is a - string value representing the REST resource this\n object represents. - Servers may infer this from the endpoint the client\n submits requests - to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'\n - \ type: string\n metadata:\n type: object\n spec:\n - \ description: IPAMConfigSpec contains the specification for an IPAMConfig\n - \ resource.\n properties:\n autoAllocateBlocks:\n - \ type: boolean\n maxBlocksPerHost:\n description: - MaxBlocksPerHost, if non-zero, is the max number of blocks\n that - can be affine to each host.\n maximum: 2147483647\n minimum: - 0\n type: integer\n strictAffinity:\n type: - boolean\n required:\n - autoAllocateBlocks\n - - strictAffinity\n type: object\n type: object\n served: true\n - \ storage: true\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n - \ conditions: []\n storedVersions: []\n" -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cni-${CLUSTER_NAME}-calico-windows - namespace: default ---- -apiVersion: v1 -data: - csi-proxy: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: csi-proxy - name: csi-proxy - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: csi-proxy - template: - metadata: - labels: - k8s-app: csi-proxy - spec: - nodeSelector: - "kubernetes.io/os": windows - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\SYSTEM" - hostNetwork: true - containers: - - name: csi-proxy - image: ghcr.io/kubernetes-sigs/sig-windows/csi-proxy:v1.0.2 -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: csi-proxy-addon - namespace: default ---- -apiVersion: v1 -data: - containerd-windows-logger: | - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - k8s-app: containerd-logger - name: containerd-logger - namespace: kube-system - spec: - selector: - matchLabels: - k8s-app: containerd-logger - template: - metadata: - labels: - k8s-app: containerd-logger - spec: - securityContext: - windowsOptions: - hostProcess: true - runAsUserName: "NT AUTHORITY\\system" - hostNetwork: true - containers: - - image: ghcr.io/kubernetes-sigs/sig-windows/eventflow-logger:v0.1.0 - args: [ "config.json" ] - name: containerd-logger - imagePullPolicy: Always - volumeMounts: - - name: containerd-logger-config - mountPath: /config.json - subPath: config.json - nodeSelector: - kubernetes.io/os: windows - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: Exists - volumes: - - configMap: - name: containerd-logger-config - name: containerd-logger-config - updateStrategy: - type: RollingUpdate - --- - kind: ConfigMap - apiVersion: v1 - metadata: - name: containerd-logger-config - namespace: kube-system - data: - config.json: | - { - "inputs": [ - { - "type": "ETW", - "sessionNamePrefix": "containerd", - "cleanupOldSessions": true, - "reuseExistingSession": true, - "providers": [ - { - "providerName": "Microsoft.Virtualization.RunHCS", - "providerGuid": "0B52781F-B24D-5685-DDF6-69830ED40EC3", - "level": "Verbose" - }, - { - "providerName": "ContainerD", - "providerGuid": "2acb92c0-eb9b-571a-69cf-8f3410f383ad", - "level": "Verbose" - } - ] - } - ], - "filters": [ - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::LayerID && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == hcsshim::NameToGuid && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.Stats && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == containerd.task.v2.Task.State && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetProcessProperties && hasnoproperty error" - }, - { - "type": "drop", - "include": "ProviderName == Microsoft.Virtualization.RunHCS && name == HcsGetComputeSystemProperties && hasnoproperty error" - } - ], - "outputs": [ - { - "type": "StdOutput" - } - ], - "schemaVersion": "2016-08-11" - } -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: containerd-logger-${CLUSTER_NAME} - namespace: default diff --git a/templates/test/ci/cluster-template-prow-ipv6.yaml b/templates/test/ci/cluster-template-prow-ipv6.yaml index 9ec986884dc..f5f4a45672b 100644 --- a/templates/test/ci/cluster-template-prow-ipv6.yaml +++ b/templates/test/ci/cluster-template-prow-ipv6.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico-ipv6 name: ${CLUSTER_NAME} namespace: default @@ -388,3 +389,52 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} diff --git a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml index a7c01f970a3..7c5df0d8924 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-ci-version.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -637,6 +638,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml index 4e6f89b03fe..8a5118a6d88 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -408,6 +409,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-machine-pool.yaml b/templates/test/ci/cluster-template-prow-machine-pool.yaml index 1291a26ff24..eedf5608f58 100644 --- a/templates/test/ci/cluster-template-prow-machine-pool.yaml +++ b/templates/test/ci/cluster-template-prow-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -402,6 +403,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml index 053672331e6..5480a1e87cf 100644 --- a/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml +++ b/templates/test/ci/cluster-template-prow-nvidia-gpu.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default @@ -266,3 +267,52 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index bd2f4ed9c7b..e96c1a206c1 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default @@ -86,26 +87,14 @@ spec: clusterConfiguration: apiServer: extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true + cloud-provider: external timeoutForControlPlane: 20m controllerManager: extraArgs: allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external cluster-name: ${CLUSTER_NAME} v: "4" - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true etcd: local: dataDir: /var/lib/etcddisk/etcd @@ -140,15 +129,13 @@ spec: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' joinConfiguration: nodeRegistration: kubeletExtraArgs: azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' mounts: - - LABEL=etcd_disk @@ -249,8 +236,7 @@ spec: joinConfiguration: nodeRegistration: kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure + cloud-provider: external name: '{{ ds.meta_data["local_hostname"] }}' preKubeadmCommands: [] --- @@ -335,6 +321,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: resources: | diff --git a/templates/test/ci/cluster-template-prow-topology.yaml b/templates/test/ci/cluster-template-prow-topology.yaml index d860398a0d9..1c8e0f743ac 100644 --- a/templates/test/ci/cluster-template-prow-topology.yaml +++ b/templates/test/ci/cluster-template-prow-topology.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -133,6 +134,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: proxy: | diff --git a/templates/test/ci/cluster-template-prow-workload-identity.yaml b/templates/test/ci/cluster-template-prow-workload-identity.yaml index 62a5c7e8aa3..cc6e6f02b9e 100644 --- a/templates/test/ci/cluster-template-prow-workload-identity.yaml +++ b/templates/test/ci/cluster-template-prow-workload-identity.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default @@ -272,3 +273,52 @@ spec: runOnControlPlane: true windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index e0a0e625a7c..25c5c2ea1cd 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -465,6 +466,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: proxy: | diff --git a/test/e2e/data/infrastructure-azure/v1beta1/patches/cluster-label-calico.yaml b/templates/test/ci/patches/cluster-label-cloud-provider-azure-flatcar.yaml similarity index 74% rename from test/e2e/data/infrastructure-azure/v1beta1/patches/cluster-label-calico.yaml rename to templates/test/ci/patches/cluster-label-cloud-provider-azure-flatcar.yaml index 9fc5277a259..5ef9c032813 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/patches/cluster-label-calico.yaml +++ b/templates/test/ci/patches/cluster-label-cloud-provider-azure-flatcar.yaml @@ -3,4 +3,4 @@ kind: Cluster metadata: name: ${CLUSTER_NAME} labels: - cni: calico \ No newline at end of file + cloud-provider: azure-flatcar \ No newline at end of file diff --git a/templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml b/templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml new file mode 100644 index 00000000000..94c474ff97f --- /dev/null +++ b/templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml @@ -0,0 +1,6 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ${CLUSTER_NAME} + labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} \ No newline at end of file diff --git a/templates/test/ci/prow-azure-cni-v1/kustomization.yaml b/templates/test/ci/prow-azure-cni-v1/kustomization.yaml index edc3ef31580..3d9aae49c58 100644 --- a/templates/test/ci/prow-azure-cni-v1/kustomization.yaml +++ b/templates/test/ci/prow-azure-cni-v1/kustomization.yaml @@ -4,6 +4,9 @@ namespace: default resources: - ../../../flavors/azure-cni-v1/ - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml diff --git a/templates/test/ci/prow-custom-vnet/kustomization.yaml b/templates/test/ci/prow-custom-vnet/kustomization.yaml index 3560a6b1671..c8bcc156593 100644 --- a/templates/test/ci/prow-custom-vnet/kustomization.yaml +++ b/templates/test/ci/prow-custom-vnet/kustomization.yaml @@ -6,6 +6,8 @@ resources: - ../prow/mhc.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/mhc.yaml @@ -14,3 +16,4 @@ patchesStrategicMerge: - ../patches/uami-md-0.yaml - ../patches/uami-control-plane.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml diff --git a/templates/test/ci/prow-dual-stack/kustomization.yaml b/templates/test/ci/prow-dual-stack/kustomization.yaml index b3a9446fcd6..f2dc50c53ff 100644 --- a/templates/test/ci/prow-dual-stack/kustomization.yaml +++ b/templates/test/ci/prow-dual-stack/kustomization.yaml @@ -6,9 +6,12 @@ resources: - machine-pool-dualstack.yaml - ../../../addons/cluster-api-helm/calico-dual-stack.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - patches/azure-machine-template-control-plane.yaml - patches/azure-machine-template.yaml - patches/cluster-label-calico-dual-stack.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml diff --git a/templates/test/ci/prow-edgezone/kustomization.yaml b/templates/test/ci/prow-edgezone/kustomization.yaml index b8305b908a1..5f98238048b 100644 --- a/templates/test/ci/prow-edgezone/kustomization.yaml +++ b/templates/test/ci/prow-edgezone/kustomization.yaml @@ -5,6 +5,8 @@ resources: - ../../../flavors/edgezone - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml @@ -16,3 +18,5 @@ patchesStrategicMerge: - patches/machine-type.yaml - patches/kubernetes-version.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml + diff --git a/templates/test/ci/prow-flatcar/kustomization.yaml b/templates/test/ci/prow-flatcar/kustomization.yaml index 2934efaafa3..1e6f735a8ca 100644 --- a/templates/test/ci/prow-flatcar/kustomization.yaml +++ b/templates/test/ci/prow-flatcar/kustomization.yaml @@ -5,7 +5,9 @@ resources: - ../../../flavors/flatcar/ - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-flatcar.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure-flatcar.yaml diff --git a/templates/test/ci/prow-intree-cloud-provider-machine-pool/kustomization.yaml b/templates/test/ci/prow-intree-cloud-provider-machine-pool/kustomization.yaml deleted file mode 100644 index f7bc246d986..00000000000 --- a/templates/test/ci/prow-intree-cloud-provider-machine-pool/kustomization.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../prow-machine-pool -patchesStrategicMerge: - - ../prow-intree-cloud-provider/patches/intree-cp.yaml - - patches/intree-mp.yaml diff --git a/templates/test/ci/prow-intree-cloud-provider-machine-pool/patches/intree-mp.yaml b/templates/test/ci/prow-intree-cloud-provider-machine-pool/patches/intree-mp.yaml deleted file mode 100644 index 92e993895c0..00000000000 --- a/templates/test/ci/prow-intree-cloud-provider-machine-pool/patches/intree-mp.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default -spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfig -metadata: - name: ${CLUSTER_NAME}-mp-win - namespace: default -spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - pod-infra-container-image: mcr.microsoft.com/oss/kubernetes/pause:3.4.1 - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - diff --git a/templates/test/ci/prow-intree-cloud-provider/kustomization.yaml b/templates/test/ci/prow-intree-cloud-provider/kustomization.yaml deleted file mode 100644 index fc5e636076e..00000000000 --- a/templates/test/ci/prow-intree-cloud-provider/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: default -resources: - - ../prow -patchesStrategicMerge: - - patches/intree-cp.yaml - - patches/intree-md-0.yaml - - patches/intree-md-win.yaml \ No newline at end of file diff --git a/templates/test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml b/templates/test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml deleted file mode 100644 index 78bcc083644..00000000000 --- a/templates/test/ci/prow-intree-cloud-provider/patches/intree-cp.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane - namespace: default -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - controllerManager: - extraArgs: - allocate-node-cidrs: "false" - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - cluster-name: ${CLUSTER_NAME} - extraVolumes: - - hostPath: /etc/kubernetes/azure.json - mountPath: /etc/kubernetes/azure.json - name: cloud-config - readOnly: true - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - azure-container-registry-config: /etc/kubernetes/azure.json - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure diff --git a/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml b/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml deleted file mode 100644 index 36aeceb28f8..00000000000 --- a/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-0.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 - namespace: default -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-config: /etc/kubernetes/azure.json - cloud-provider: azure diff --git a/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-win.yaml b/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-win.yaml deleted file mode 100644 index 8514a5bf856..00000000000 --- a/templates/test/ci/prow-intree-cloud-provider/patches/intree-md-win.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-win - namespace: default -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - v: "2" - windows-priorityclass: ABOVE_NORMAL_PRIORITY_CLASS - azure-container-registry-config: c:/k/azure.json - cloud-config: c:/k/azure.json - cloud-provider: azure - diff --git a/templates/test/ci/prow-ipv6/kustomization.yaml b/templates/test/ci/prow-ipv6/kustomization.yaml index dc8be660a23..41c86f27d8b 100644 --- a/templates/test/ci/prow-ipv6/kustomization.yaml +++ b/templates/test/ci/prow-ipv6/kustomization.yaml @@ -6,7 +6,10 @@ resources: - machine-pool-ipv6.yaml - ../../../addons/cluster-api-helm/calico-ipv6.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - patches/cluster-label-calico-ipv6.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml diff --git a/templates/test/ci/prow-machine-pool/kustomization.yaml b/templates/test/ci/prow-machine-pool/kustomization.yaml index 7d11304870f..4a9689db6c7 100644 --- a/templates/test/ci/prow-machine-pool/kustomization.yaml +++ b/templates/test/ci/prow-machine-pool/kustomization.yaml @@ -8,6 +8,8 @@ resources: - ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/azuremachinepool-vmextension.yaml - ../patches/tags.yaml @@ -15,6 +17,7 @@ patchesStrategicMerge: - ../patches/machine-pool-worker-counts.yaml - ../patches/windows-containerd-labels.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml configMapGenerator: - name: cni-${CLUSTER_NAME}-calico-windows files: diff --git a/templates/test/ci/prow-nvidia-gpu/kustomization.yaml b/templates/test/ci/prow-nvidia-gpu/kustomization.yaml index e9fbee8abb8..863832e74b7 100644 --- a/templates/test/ci/prow-nvidia-gpu/kustomization.yaml +++ b/templates/test/ci/prow-nvidia-gpu/kustomization.yaml @@ -5,11 +5,14 @@ resources: - ../../../flavors/nvidia-gpu - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml - ../patches/azurecluster-gpu.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml patches: - path: patches/node-storage-type.yaml target: diff --git a/templates/test/ci/prow-private/kustomization.yaml b/templates/test/ci/prow-private/kustomization.yaml index a58c55dc527..0c5aa788163 100644 --- a/templates/test/ci/prow-private/kustomization.yaml +++ b/templates/test/ci/prow-private/kustomization.yaml @@ -6,6 +6,8 @@ resources: - cni-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/controller-manager.yaml @@ -13,9 +15,8 @@ patchesStrategicMerge: - patches/vnet-peerings.yaml - ../patches/uami-md-0.yaml - ../patches/uami-control-plane.yaml - - ../prow-intree-cloud-provider/patches/intree-cp.yaml # TODO: remove once CAPI supports Helm addons - - ../prow-intree-cloud-provider/patches/intree-md-0.yaml # TODO: remove once CAPI supports Helm addons - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml patches: - path: patches/user-assigned.yaml target: diff --git a/templates/test/ci/prow-topology/kustomization.yaml b/templates/test/ci/prow-topology/kustomization.yaml index 87ad2db29b2..7b684347cce 100644 --- a/templates/test/ci/prow-topology/kustomization.yaml +++ b/templates/test/ci/prow-topology/kustomization.yaml @@ -7,10 +7,13 @@ resources: - ../../../addons/windows/csi-proxy/csi-proxy-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/windows-containerd-labels.yaml - cluster.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml configMapGenerator: - name: cni-${CLUSTER_NAME}-calico-windows files: diff --git a/templates/test/ci/prow-workload-identity/kustomization.yaml b/templates/test/ci/prow-workload-identity/kustomization.yaml index 3859eae26a5..f50d45ae608 100644 --- a/templates/test/ci/prow-workload-identity/kustomization.yaml +++ b/templates/test/ci/prow-workload-identity/kustomization.yaml @@ -5,6 +5,8 @@ resources: - ../../../flavors/default - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/azureclusteridentity-azwi.yaml - ../patches/tags.yaml @@ -13,3 +15,5 @@ patchesStrategicMerge: - ../patches/uami-md-0.yaml - ../patches/uami-control-plane.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml + diff --git a/templates/test/ci/prow/kustomization.yaml b/templates/test/ci/prow/kustomization.yaml index ebb1c2ead07..a1414705aaa 100644 --- a/templates/test/ci/prow/kustomization.yaml +++ b/templates/test/ci/prow/kustomization.yaml @@ -12,6 +12,8 @@ resources: - ../../../addons/windows/containerd-logging/containerd-logger-resource-set.yaml - ../../../addons/cluster-api-helm/calico.yaml - ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure.yaml + - ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ../patches/tags.yaml - ../patches/mhc.yaml @@ -23,6 +25,7 @@ patchesStrategicMerge: - ../patches/windows-containerd-labels.yaml - ../patches/windows-server-version.yaml - ../patches/cluster-label-calico.yaml + - ../patches/cluster-label-cloud-provider-azure.yaml patches: - target: group: bootstrap.cluster.x-k8s.io diff --git a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml index c76c5216966..eff5640c41a 100644 --- a/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml +++ b/templates/test/dev/cluster-template-custom-builds-machine-pool.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -585,6 +586,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index a9fb50514ff..cdb931ca874 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -2,6 +2,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico cni-windows: ${CLUSTER_NAME}-calico containerd-logger: enabled @@ -677,6 +678,55 @@ spec: windows: useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci + namespace: default +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: v1 data: kube-proxy-patch: |- diff --git a/test/e2e/azure_privatecluster.go b/test/e2e/azure_privatecluster.go index f5011dc1b66..780a39eef32 100644 --- a/test/e2e/azure_privatecluster.go +++ b/test/e2e/azure_privatecluster.go @@ -129,8 +129,6 @@ func AzurePrivateClusterSpec(ctx context.Context, inputGetter func() AzurePrivat Expect(os.Setenv(AzureBastionSubnetCidr, "10.255.255.224/27")).To(Succeed()) result := &clusterctl.ApplyClusterTemplateAndWaitResult{} - // NOTE: We don't add control plane waiters here because Helm install will fail since the apiserver is private and not reachable from the prow cluster. - // As a workaround, we use in-tree cloud-provider-azure on the private cluster until a Helm integration is available. clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( specName, withClusterProxy(publicClusterProxy), diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 5c3039cd97c..6a54343749f 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -649,68 +649,6 @@ var _ = Describe("Workload cluster creation", func() { }) }) - // ci-e2e.sh and Prow CI skip this test by default. To include this test, set `GINKGO_SKIP=""`. - Context("Creating a cluster that uses the intree cloud provider [OPTIONAL]", func() { - It("with a 1 control plane nodes and 2 worker nodes", func() { - By("using user-assigned identity") - clusterName = getClusterName(clusterNamePrefix, "intree") - clusterctl.ApplyClusterTemplateAndWait(ctx, createApplyClusterTemplateInput( - specName, - withFlavor("intree-cloud-provider"), - withNamespace(namespace.Name), - withClusterName(clusterName), - withControlPlaneMachineCount(1), - withWorkerMachineCount(2), - withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{ - WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons, - }), - withPostMachinesProvisioned(func() { - EnsureDaemonsets(ctx, func() DaemonsetsSpecInput { - return DaemonsetsSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - } - }) - }), - ), result) - - By("Verifying expected VM extensions are present on the node", func() { - AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { - return AzureVMExtensionsSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - } - }) - }) - - By("Creating an accessible load balancer", func() { - AzureLBSpec(ctx, func() AzureLBSpecInput { - return AzureLBSpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - SkipCleanup: skipCleanup, - } - }) - }) - - By("Creating a deployment that uses persistent volume", func() { - AzureDiskCSISpec(ctx, func() AzureDiskCSISpecInput { - return AzureDiskCSISpecInput{ - BootstrapClusterProxy: bootstrapClusterProxy, - Namespace: namespace, - ClusterName: clusterName, - SkipCleanup: skipCleanup, - } - }) - }) - - By("PASSED!") - }) - }) - // You can override the default SKU `Standard_D2s_v3` by setting the // `AZURE_AKS_NODE_MACHINE_TYPE` environment variable. Context("Creating an AKS cluster [Managed Kubernetes]", func() { diff --git a/test/e2e/cloud-provider-azure.go b/test/e2e/cloud-provider-azure.go index 4c32bdda79a..489ccea05ad 100644 --- a/test/e2e/cloud-provider-azure.go +++ b/test/e2e/cloud-provider-azure.go @@ -42,34 +42,39 @@ const ( azureDiskCSIDriverCAAPHLabelName = "azuredisk-csi" ) -// InstallCNIAndCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart +// EnsureCNIAndCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart // and a CNI and validates that expected pods exist and are Ready. -func InstallCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, installHelmChart bool, cidrBlocks []string, hasWindows bool) { - specName := "cloud-provider-azure-install" - By("Installing cloud-provider-azure components via helm") - options := &HelmOptions{ - Values: []string{ - fmt.Sprintf("infra.clusterName=%s", input.ClusterName), - "cloudControllerManager.logVerbosity=4", - }, - StringValues: []string{fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `\,`))}, - } - // If testing a CI version of Kubernetes, use CCM and CNM images built from source. - if useCIArtifacts || usePRArtifacts { - options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageName=%s", os.Getenv("CCM_IMAGE_NAME"))) - options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageName=%s", os.Getenv("CNM_IMAGE_NAME"))) - options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) - options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) - options.StringValues = append(options.StringValues, fmt.Sprintf("cloudControllerManager.imageTag=%s", os.Getenv("IMAGE_TAG_CCM"))) - options.StringValues = append(options.StringValues, fmt.Sprintf("cloudNodeManager.imageTag=%s", os.Getenv("IMAGE_TAG_CNM"))) - } +func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, installHelmChart bool, cidrBlocks []string, hasWindows bool) { + specName := "ensure-cloud-provider-azure" + clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName) - if strings.Contains(input.ClusterName, "flatcar") { - options.StringValues = append(options.StringValues, "cloudControllerManager.caCertDir=/usr/share/ca-certificates") - } + if installHelmChart { + By("Installing cloud-provider-azure components via helm") + options := &HelmOptions{ + Values: []string{ + fmt.Sprintf("infra.clusterName=%s", input.ClusterName), + "cloudControllerManager.logVerbosity=4", + }, + StringValues: []string{fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `\,`))}, + } + // If testing a CI version of Kubernetes, use CCM and CNM images built from source. + if useCIArtifacts || usePRArtifacts { + options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageName=%s", os.Getenv("CCM_IMAGE_NAME"))) + options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageName=%s", os.Getenv("CNM_IMAGE_NAME"))) + options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) + options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY"))) + options.StringValues = append(options.StringValues, fmt.Sprintf("cloudControllerManager.imageTag=%s", os.Getenv("IMAGE_TAG_CCM"))) + options.StringValues = append(options.StringValues, fmt.Sprintf("cloudNodeManager.imageTag=%s", os.Getenv("IMAGE_TAG_CNM"))) + } - clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName) - InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options, "") + if strings.Contains(input.ClusterName, "flatcar") { + options.StringValues = append(options.StringValues, "cloudControllerManager.caCertDir=/usr/share/ca-certificates") + } + + InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options, "") + } else { + By("Ensuring cloud-provider-azure is installed via CAAPH") + } // We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized. EnsureCNI(ctx, input, installHelmChart, cidrBlocks, hasWindows) diff --git a/test/e2e/common.go b/test/e2e/common.go index 9cee7751c4f..6eda7e293ac 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -292,7 +292,7 @@ func ensureControlPlaneInitialized(ctx context.Context, input clusterctl.ApplyCu if kubeadmControlPlane.Spec.KubeadmConfigSpec.ClusterConfiguration.ControllerManager.ExtraArgs["cloud-provider"] != infrav1.AzureNetworkPluginName { // There is a co-dependency between cloud-provider and CNI so we install both together if cloud-provider is external. - InstallCNIAndCloudProviderAzureHelmChart(ctx, input, installHelmCharts, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) + EnsureCNIAndCloudProviderAzureHelmChart(ctx, input, installHelmCharts, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) } else { EnsureCNI(ctx, input, installHelmCharts, cluster.Spec.ClusterNetwork.Pods.CIDRBlocks, hasWindows) } diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index 1b40cdef8a7..df2ffa1413e 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -124,8 +124,6 @@ providers: targetName: "cluster-template-conformance-ci-artifacts-dual-stack.yaml" - sourcePath: "${PWD}/templates/test/dev/cluster-template-custom-builds.yaml" targetName: "cluster-template-conformance-presubmit-artifacts.yaml" - - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml" - targetName: "cluster-template-intree-cloud-provider.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-machine-pool-flex.yaml" targetName: "cluster-template-machine-pool-flex.yaml" - sourcePath: "${PWD}/templates/test/ci/cluster-template-prow-workload-identity.yaml" diff --git a/test/e2e/conformance_test.go b/test/e2e/conformance_test.go index 774e534f135..0d92f9600fa 100644 --- a/test/e2e/conformance_test.go +++ b/test/e2e/conformance_test.go @@ -110,6 +110,7 @@ var _ = Describe("Conformance Tests", func() { kubernetesVersion, err = resolveCIVersion(kubernetesVersion) Expect(err).NotTo(HaveOccurred()) Expect(os.Setenv("CI_VERSION", kubernetesVersion)).To(Succeed()) + Expect(os.Setenv("CLOUD_PROVIDER_AZURE_LABEL", "azure-ci")).To(Succeed()) if useCIArtifacts { flavor = "conformance-ci-artifacts" diff --git a/test/e2e/data/infrastructure-azure/v1beta1/bases/azure-cluster-identity.yaml b/test/e2e/data/infrastructure-azure/v1beta1/bases/azure-cluster-identity.yaml deleted file mode 100644 index fed37c08c34..00000000000 --- a/test/e2e/data/infrastructure-azure/v1beta1/bases/azure-cluster-identity.yaml +++ /dev/null @@ -1,13 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureClusterIdentity -metadata: - name: "${CLUSTER_IDENTITY_NAME}" - labels: - clusterctl.cluster.x-k8s.io/move-hierarchy: "true" -spec: - type: ServicePrincipal - allowedNamespaces: {} - tenantID: "${AZURE_TENANT_ID}" - clientID: "${AZURE_CLIENT_ID}" - clientSecret: {"name":"${AZURE_CLUSTER_IDENTITY_SECRET_NAME}","namespace":"${AZURE_CLUSTER_IDENTITY_SECRET_NAMESPACE}"} diff --git a/test/e2e/data/infrastructure-azure/v1beta1/bases/azuredisk-csi-driver.yaml b/test/e2e/data/infrastructure-azure/v1beta1/bases/azuredisk-csi-driver.yaml deleted file mode 100644 index 07afde0faf6..00000000000 --- a/test/e2e/data/infrastructure-azure/v1beta1/bases/azuredisk-csi-driver.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: azuredisk-csi-driver-chart -spec: - clusterSelector: - matchLabels: - azuredisk-csi: "true" - repoURL: https://raw.githubusercontent.com/kubernetes-sigs/azuredisk-csi-driver/master/charts - chartName: azuredisk-csi-driver - releaseName: azuredisk-csi-driver-oot - namespace: kube-system - valuesTemplate: | - controller: - replicas: 1 - runOnControlPlane: true - windows: - useHostProcessContainers: {{ hasKey .Cluster.metadata.labels "cni-windows" }} \ No newline at end of file diff --git a/test/e2e/data/infrastructure-azure/v1beta1/bases/calico.yaml b/test/e2e/data/infrastructure-azure/v1beta1/bases/calico.yaml deleted file mode 100644 index ba19537703d..00000000000 --- a/test/e2e/data/infrastructure-azure/v1beta1/bases/calico.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: addons.cluster.x-k8s.io/v1alpha1 -kind: HelmChartProxy -metadata: - name: calico -spec: - clusterSelector: - matchLabels: - cni: calico - repoURL: https://docs.tigera.io/calico/charts - chartName: tigera-operator - version: ${CALICO_VERSION} - releaseName: projectcalico - namespace: tigera-operator - valuesTemplate: | - installation: - cni: - type: Calico - calicoNetwork: - bgp: Disabled - mtu: 1350 - ipPools: - ipPools:{{range $i, $cidr := .Cluster.spec.clusterNetwork.pods.cidrBlocks }} - - cidr: {{ $cidr }} - encapsulation: VXLAN{{end}} - registry: mcr.microsoft.com/oss - # Image and registry configuration for the tigera/operator pod. - tigeraOperator: - image: tigera/operator - registry: mcr.microsoft.com/oss - calicoctl: - image: mcr.microsoft.com/oss/calico/ctl \ No newline at end of file diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml index 2f539c072e4..1c30a0928ff 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation.yaml @@ -49,6 +49,53 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml index dcedaf967eb..51627e8f298 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-remediation/kustomization.yaml @@ -1,11 +1,13 @@ bases: - - ../bases/cluster-with-kcp.yaml - - ../bases/md.yaml - - mhc.yaml - - ../bases/azure-cluster-identity.yaml - - ../bases/calico.yaml - - ../bases/azuredisk-csi-driver.yaml +- ../bases/cluster-with-kcp.yaml +- ../bases/md.yaml +- mhc.yaml +- ../../../../../../templates/azure-cluster-identity/azure-cluster-identity.yaml +- ../../../../../../templates/addons/cluster-api-helm/calico.yaml +- ../../../../../../templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - - ../patches/azurecluster-identity-ref.yaml - - ../patches/cluster-label-calico.yaml +- ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-calico.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml index 3c57cfac7fe..b94216f82d6 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in.yaml @@ -49,6 +49,53 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: @@ -75,6 +122,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml index 9ff2ab9b255..839becbc484 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-kcp-scale-in/kustomization.yaml @@ -1,11 +1,14 @@ bases: - ../bases/cluster-with-kcp.yaml - ../bases/md.yaml -- ../bases/azure-cluster-identity.yaml -- ../bases/calico.yaml -- ../bases/azuredisk-csi-driver.yaml +- ../../../../../../templates/azure-cluster-identity/azure-cluster-identity.yaml +- ../../../../../../templates/addons/cluster-api-helm/calico.yaml +- ../../../../../../templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ./cluster-with-kcp.yaml -- ../patches/azurecluster-identity-ref.yaml -- ../patches/cluster-label-calico.yaml +- ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-calico.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml index 0e4bb3d61b3..1ecb15fdea8 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool.yaml @@ -49,6 +49,53 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfig metadata: @@ -73,6 +120,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml index 47616e11714..ace1f63adb7 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-machine-pool/kustomization.yaml @@ -1,10 +1,13 @@ resources: - - ../bases/cluster-with-kcp.yaml - - ../bases/mp.yaml - - ../bases/azure-cluster-identity.yaml - - ../bases/calico.yaml - - ../bases/azuredisk-csi-driver.yaml +- ../bases/cluster-with-kcp.yaml +- ../bases/mp.yaml +- ../../../../../../templates/azure-cluster-identity/azure-cluster-identity.yaml +- ../../../../../../templates/addons/cluster-api-helm/calico.yaml +- ../../../../../../templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - - ../patches/azurecluster-identity-ref.yaml - - ../patches/cluster-label-calico.yaml +- ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-calico.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml index 4a3487f8491..702dde4105c 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation.yaml @@ -49,6 +49,53 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: @@ -75,6 +122,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml index b8b5c7bf1d8..1de86072e7a 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-md-remediation/kustomization.yaml @@ -1,12 +1,15 @@ bases: - - ../bases/cluster-with-kcp.yaml - - ../bases/md.yaml - - mhc.yaml - - ../bases/azure-cluster-identity.yaml - - ../bases/calico.yaml - - ../bases/azuredisk-csi-driver.yaml +- ../bases/cluster-with-kcp.yaml +- ../bases/md.yaml +- mhc.yaml +- ../../../../../../templates/azure-cluster-identity/azure-cluster-identity.yaml +- ../../../../../../templates/addons/cluster-api-helm/calico.yaml +- ../../../../../../templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ./md.yaml -- ../patches/azurecluster-identity-ref.yaml -- ../patches/cluster-label-calico.yaml +- ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-calico.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml index 44cd6f4da38..5e48d37795b 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain.yaml @@ -49,6 +49,53 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: @@ -75,6 +122,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml index 068b84ec7db..fe6f31be6d0 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-node-drain/kustomization.yaml @@ -1,12 +1,15 @@ bases: - ../bases/cluster-with-kcp.yaml - ../bases/md.yaml -- ../bases/azure-cluster-identity.yaml -- ../bases/calico.yaml -- ../bases/azuredisk-csi-driver.yaml +- ../../../../../../templates/azure-cluster-identity/azure-cluster-identity.yaml +- ../../../../../../templates/addons/cluster-api-helm/calico.yaml +- ../../../../../../templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: - ./md.yaml - ./cluster-with-kcp.yaml -- ../patches/azurecluster-identity-ref.yaml -- ../patches/cluster-label-calico.yaml +- ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-calico.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades/kustomization.yaml index fee14fe866f..5cb009e75f2 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template-upgrades/kustomization.yaml @@ -2,7 +2,7 @@ resources: - ../bases/cluster-with-kcp.yaml - ../bases/md.yaml - ../bases/mp.yaml - - ../bases/azure-cluster-identity.yaml + - ../../../../../../templates/azure-cluster-identity/azure-cluster-identity.yaml patchesStrategicMerge: - - ../patches/azurecluster-identity-ref.yaml + - ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml index 42e90f3c501..139054cfc0a 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template.yaml @@ -49,6 +49,53 @@ spec: image: mcr.microsoft.com/oss/calico/ctl version: ${CALICO_VERSION} --- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + logVerbosity: 4 +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmChartProxy +metadata: + name: cloud-provider-azure-chart-ci +spec: + chartName: cloud-provider-azure + clusterSelector: + matchLabels: + cloud-provider: azure-ci + releaseName: cloud-provider-azure-oot + repoURL: https://raw.githubusercontent.com/kubernetes-sigs/cloud-provider-azure/master/helm/repo + valuesTemplate: |- + infra: + clusterName: {{ .Cluster.metadata.name }} + cloudControllerManager: + cloudConfig: ${CLOUD_CONFIG:-"/etc/kubernetes/azure.json"} + cloudConfigSecretName: ${CONFIG_SECRET_NAME:-""} + clusterCIDR: {{ .Cluster.spec.clusterNetwork.pods.cidrBlocks | join "," }} + imageName: ${CCM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CCM:-""} + logVerbosity: ${CCM_LOG_VERBOSITY:-4} + replicas: ${CCM_COUNT:-1} + enableDynamicReloading: ${ENABLE_DYNAMIC_RELOADING:-false} + cloudNodeManager: + imageName: ${CNM_IMAGE_NAME:-""} + imageRepository: ${IMAGE_REGISTRY:-""} + imageTag: ${IMAGE_TAG_CNM:-""} +--- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: @@ -75,6 +122,7 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: labels: + cloud-provider: ${CLOUD_PROVIDER_AZURE_LABEL:=azure} cni: calico name: ${CLUSTER_NAME} namespace: default diff --git a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml index 12b0a40d8fa..25b33489267 100644 --- a/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml +++ b/test/e2e/data/infrastructure-azure/v1beta1/cluster-template/kustomization.yaml @@ -1,10 +1,13 @@ bases: - ../bases/cluster-with-kcp.yaml - ../bases/md.yaml -- ../bases/azure-cluster-identity.yaml -- ../bases/calico.yaml -- ../bases/azuredisk-csi-driver.yaml +- ../../../../../../templates/azure-cluster-identity/azure-cluster-identity.yaml +- ../../../../../../templates/addons/cluster-api-helm/calico.yaml +- ../../../../../../templates/addons/cluster-api-helm/azuredisk-csi-driver.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure.yaml +- ../../../../../../templates/addons/cluster-api-helm/cloud-provider-azure-ci.yaml patchesStrategicMerge: -- ../patches/azurecluster-identity-ref.yaml -- ../patches/cluster-label-calico.yaml +- ../../../../../../templates/azure-cluster-identity/azurecluster-identity-ref.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-calico.yaml +- ../../../../../../templates/test/ci/patches/cluster-label-cloud-provider-azure.yaml diff --git a/test/e2e/data/infrastructure-azure/v1beta1/patches/azurecluster-identity-ref.yaml b/test/e2e/data/infrastructure-azure/v1beta1/patches/azurecluster-identity-ref.yaml deleted file mode 100644 index 9ec26da18c9..00000000000 --- a/test/e2e/data/infrastructure-azure/v1beta1/patches/azurecluster-identity-ref.yaml +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 -kind: AzureCluster -metadata: - name: ${CLUSTER_NAME} -spec: - identityRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 - kind: AzureClusterIdentity - name: "${CLUSTER_IDENTITY_NAME}"