Nutanix Failure Domains for Worker nodes

config/crd/bases/anywhere.eks.amazonaws.com_nutanixdatacenterconfigs.yaml

@@ -115,6 +115,12 @@ spec:
                         - type
                         type: object
                       type: array
+                    workerMachineGroups:
+                      description: Worker Machine Groups holds the list of worker
+                        machine group names that will use this failure domain.
+                      items:
+                        type: string
+                      type: array
                   required:
                   - name
                   type: object

pkg/api/v1alpha1/nutanixdatacenterconfig_types.go

@@ -68,6 +68,10 @@ type NutanixDatacenterFailureDomain struct {
 	// Subnets holds the list of subnets identifiers cluster's network subnets.
 	// +kubebuilder:validation:Required
 	Subnets []NutanixResourceIdentifier `json:"subnets,omitempty"`
+
+	// Worker Machine Groups holds the list of worker machine group names that will use this failure domain.
+	// +optional
+	WorkerMachineGroups []string `json:"workerMachineGroups,omitempty"`
 }
 
 // NutanixDatacenterConfigStatus defines the observed state of NutanixDatacenterConfig.
@@ -165,7 +169,7 @@ func (in *NutanixDatacenterConfig) Validate() error {
 		}
 	}
 
-	if in.Spec.FailureDomains != nil && len(in.Spec.FailureDomains) != 0 {
+	if len(in.Spec.FailureDomains) != 0 {
 		dccName := in.Namespace + "/" + in.Name
 		validateClusterResourceIdentifier := createValidateNutanixResourceFunc("NutanixDatacenterConfig.Spec.FailureDomains.Cluster", "cluster", dccName)
 		validateSubnetResourceIdentifier := createValidateNutanixResourceFunc("NutanixDatacenterConfig.Spec.FailureDomains.Subnets", "subnet", dccName)

pkg/providers/nutanix/config/md-template.yaml

@@ -1,3 +1,105 @@
+{{- if .failureDomains }}{{- range $index, $fd := .failureDomains }}
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: "{{$.clusterName}}"
+  name: "{{$.workerNodeGroupName}}-{{$fd.Name}}"
+  namespace: "{{$.eksaSystemNamespace}}"
+{{- if $.autoscalingConfig }}
+  annotations:
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "{{ $.autoscalingConfig.MinCount }}"
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "{{ $.autoscalingConfig.MaxCount }}"
+{{- end }}
+spec:
+  clusterName: "{{$.clusterName}}"
+{{- if not $.autoscalingConfig }}
+  replicas: {{ index $.failureDomainsReplicas $fd.Name }}
+{{- end }}
+  selector:
+    matchLabels: {}
+  template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/cluster-name: "{{$.clusterName}}"
+    spec:
+      failureDomain: "{{$fd.Name}}"
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: KubeadmConfigTemplate
+          name: "{{$.workloadkubeadmconfigTemplateName}}"
+      clusterName: "{{$.clusterName}}"
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+        kind: NutanixMachineTemplate
+        name: "{{$.workloadTemplateName}}-{{$fd.Name}}"
+      version: "{{$.kubernetesVersion}}"
+{{- if $.upgradeRolloutStrategy }}
+  strategy:
+    rollingUpdate:
+      maxSurge: {{$.maxSurge}}
+      maxUnavailable: {{$.maxUnavailable}}
+{{- end }}
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: NutanixMachineTemplate
+metadata:
+  name: "{{$.workloadTemplateName}}-{{$fd.Name}}"
+  namespace: "{{$.eksaSystemNamespace}}"
+spec:
+  template:
+    spec:
+      providerID: "nutanix://{{$.clusterName}}-m1"
+      vcpusPerSocket: {{$.vcpusPerSocket}}
+      vcpuSockets: {{$.vcpuSockets}}
+      memorySize: {{$.memorySize}}
+      systemDiskSize: {{$.systemDiskSize}}
+      image:
+{{- if (eq $.imageIDType "name") }}
+        type: name
+        name: "{{$.imageName}}"
+{{ else if (eq $.imageIDType "uuid") }}
+        type: uuid
+        uuid: "{{$.imageUUID}}"
+{{ end }}
+      cluster:
+{{- if (eq $fd.Cluster.Type "name") }}
+        type: name
+        name: "{{ $fd.Cluster.Name }}"
+{{- else if (eq $fd.Cluster.Type "uuid") }}
+        type: uuid
+        uuid: "{{ $fd.Cluster.UUID }}"
+{{ end }}
+      subnet:
+{{- range $subnet := $fd.Subnets }}
+{{- if (eq $subnet.Type "name") }}
+        - type: name
+          name: "{{ $subnet.Name }}"
+{{- else if (eq $subnet.Type "uuid") }}
+        - type: uuid
+          uuid: "{{ $subnet.UUID }}"
+{{- end }}
+{{- end }}
+{{- if $.projectIDType}}
+      project:
+{{- if (eq $.projectIDType "name") }}
+        type: name
+        name: "{{$.projectName}}"
+{{- else if (eq $.projectIDType "uuid") }}
+        type: uuid
+        uuid: "{{$.projectUUID}}"
+{{ end }}
+{{ end }}
+{{- if $.additionalCategories}}
+      additionalCategories:
+{{- range $.additionalCategories}}
+        - key:   "{{ $.Key }}"
+          value: "{{ $.Value }}"
+{{- end }}
+{{- end }}
+---
+{{- end }}{{- else }}
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
 metadata:
@@ -95,6 +197,7 @@ spec:
 {{- end }}
 {{- end }}
 ---
+{{- end }}
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KubeadmConfigTemplate
 metadata:

pkg/providers/nutanix/provider_test.go

@@ -312,13 +312,13 @@ func TestNutanixProviderSetupAndValidateCreate(t *testing.T) {
 			name:            "cluster config with unsupported upgrade strategy configuration for cp",
 			clusterConfFile: "testdata/cluster_nutanix_with_upgrade_strategy_cp.yaml",
 			expectErr:       true,
-			expectErrStr:    "failed setup and validations: Upgrade rollout strategy customization is not supported for nutanix provider",
+			expectErrStr:    "failed setup and validations: upgrade rollout strategy customization is not supported for nutanix provider",
 		},
 		{
 			name:            "cluster config with unsupported upgrade strategy configuration for md",
 			clusterConfFile: "testdata/cluster_nutanix_with_upgrade_strategy_md.yaml",
 			expectErr:       true,
-			expectErrStr:    "failed setup and validations: Upgrade rollout strategy customization is not supported for nutanix provider",
+			expectErrStr:    "failed setup and validations: upgrade rollout strategy customization is not supported for nutanix provider",
 		},
 	}
 
@@ -507,13 +507,13 @@ func TestNutanixProviderSetupAndValidateDeleteCluster(t *testing.T) {
 			name:            "cluster config with unsupported upgrade strategy configuration for cp",
 			clusterConfFile: "testdata/cluster_nutanix_with_upgrade_strategy_cp.yaml",
 			expectErr:       true,
-			expectErrStr:    "failed setup and validations: Upgrade rollout strategy customization is not supported for nutanix provider",
+			expectErrStr:    "failed setup and validations: upgrade rollout strategy customization is not supported for nutanix provider",
 		},
 		{
 			name:            "cluster config with unsupported upgrade strategy configuration for md",
 			clusterConfFile: "testdata/cluster_nutanix_with_upgrade_strategy_md.yaml",
 			expectErr:       true,
-			expectErrStr:    "failed setup and validations: Upgrade rollout strategy customization is not supported for nutanix provider",
+			expectErrStr:    "failed setup and validations: upgrade rollout strategy customization is not supported for nutanix provider",
 		},
 	}
 
@@ -559,13 +559,13 @@ func TestNutanixProviderSetupAndValidateUpgradeCluster(t *testing.T) {
 			name:            "cluster config with unsupported upgrade strategy configuration for cp",
 			clusterConfFile: "testdata/cluster_nutanix_with_upgrade_strategy_cp.yaml",
 			expectErr:       true,
-			expectErrStr:    "failed setup and validations: Upgrade rollout strategy customization is not supported for nutanix provider",
+			expectErrStr:    "failed setup and validations: upgrade rollout strategy customization is not supported for nutanix provider",
 		},
 		{
 			name:            "cluster config with unsupported upgrade strategy configuration for md",
 			clusterConfFile: "testdata/cluster_nutanix_with_upgrade_strategy_md.yaml",
 			expectErr:       true,
-			expectErrStr:    "failed setup and validations: Upgrade rollout strategy customization is not supported for nutanix provider",
+			expectErrStr:    "failed setup and validations: upgrade rollout strategy customization is not supported for nutanix provider",
 		},
 	}
 

pkg/providers/nutanix/template.go

@@ -346,10 +346,53 @@ func buildTemplateMapCP(
 	return values, nil
 }
 
+func calcFailureDomainReplicas(workerNodeGroupConfiguration v1alpha1.WorkerNodeGroupConfiguration, failureDomains []v1alpha1.NutanixDatacenterFailureDomain) map[string]int {
+	replicasPerFailureDomain := make(map[string]int)
+	failureDomainCount := len(failureDomains)
+
+	if workerNodeGroupConfiguration.AutoScalingConfiguration != nil {
+		return replicasPerFailureDomain
+	}
+
+	if failureDomainCount == 0 {
+		return replicasPerFailureDomain
+	}
+
+	workerNodeGroupCount := failureDomainCount
+	if workerNodeGroupConfiguration.Count != nil {
+		workerNodeGroupCount = int(*workerNodeGroupConfiguration.Count)
+	}
+
+	minCount := int(workerNodeGroupCount / failureDomainCount)
+
+	for i := 0; i < len(failureDomains); i++ {
+		replicasPerFailureDomain[failureDomains[i].Name] = minCount
+	}
+	replicasPerFailureDomain[failureDomains[0].Name] = workerNodeGroupCount - (failureDomainCount-1)*minCount
+
+	return replicasPerFailureDomain
+}
+
+func getFailureDomainsForWorkerNodeGroup(allFailureDomains []v1alpha1.NutanixDatacenterFailureDomain, workerNodeGroupConfigurationName string) []v1alpha1.NutanixDatacenterFailureDomain {
+	result := make([]v1alpha1.NutanixDatacenterFailureDomain, 0)
+	for _, fd := range allFailureDomains {
+		for _, workerMachineGroup := range fd.WorkerMachineGroups {
+			if workerMachineGroup == workerNodeGroupConfigurationName {
+				result = append(result, fd)
+			}
+		}
+	}
+
+	return result
+}
+
 func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupMachineSpec v1alpha1.NutanixMachineConfigSpec, workerNodeGroupConfiguration v1alpha1.WorkerNodeGroupConfiguration) (map[string]interface{}, error) {
 	versionsBundle := clusterSpec.WorkerNodeGroupVersionsBundle(workerNodeGroupConfiguration)
 	format := "cloud-config"
 
+	failureDomainsForWorkerNodeGroup := getFailureDomainsForWorkerNodeGroup(clusterSpec.NutanixDatacenter.Spec.FailureDomains, workerNodeGroupConfiguration.Name)
+	replicasPerFailureDomain := calcFailureDomainReplicas(workerNodeGroupConfiguration, failureDomainsForWorkerNodeGroup)
+
 	values := map[string]interface{}{
 		"clusterName":            clusterSpec.Cluster.Name,
 		"eksaSystemNamespace":    constants.EksaSystemNamespace,
@@ -374,6 +417,8 @@ func buildTemplateMapMD(clusterSpec *cluster.Spec, workerNodeGroupMachineSpec v1
 		"subnetUUID":             workerNodeGroupMachineSpec.Subnet.UUID,
 		"workerNodeGroupName":    fmt.Sprintf("%s-%s", clusterSpec.Cluster.Name, workerNodeGroupConfiguration.Name),
 		"workerNodeGroupTaints":  workerNodeGroupConfiguration.Taints,
+		"failureDomains":         failureDomainsForWorkerNodeGroup,
+		"failureDomainsReplicas": replicasPerFailureDomain,
 	}
 
 	if clusterSpec.Cluster.Spec.RegistryMirrorConfiguration != nil {

pkg/providers/nutanix/template_test.go

@@ -577,9 +577,10 @@ func TestNewNutanixTemplateBuilderProxy(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotNil(t, workerSpec)
 
-	expectedWorkersSpec, err := os.ReadFile("testdata/expected_results_proxy_md.yaml")
+	resultMdFileName := "testdata/expected_results_proxy_md.yaml"
+	expectedWorkersSpec, err := os.ReadFile(resultMdFileName)
 	require.NoError(t, err)
-	assert.Equal(t, expectedWorkersSpec, workerSpec)
+	test.AssertContentToFile(t, string(expectedWorkersSpec), resultMdFileName)
 }
 
 func TestTemplateBuilder_CertSANs(t *testing.T) {
@@ -726,6 +727,53 @@ func TestTemplateBuilderFailureDomains(t *testing.T) {
 	}
 }
 
+func TestTemplateBuilderWorkersFailureDomains(t *testing.T) {
+	for _, tc := range []struct {
+		Input    string
+		OutputCP string
+		OutputMD string
+	}{
+		{
+			Input:    "testdata/eksa-cluster-worker-fds.yaml",
+			OutputCP: "testdata/expected_results_worker_fds.yaml",
+			OutputMD: "testdata/expected_results_worker_fds_md.yaml",
+		},
+	} {
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+		machineConf := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+		workerConfs := make(map[string]anywherev1.NutanixMachineConfigSpec)
+		for _, worker := range clusterSpec.Cluster.Spec.WorkerNodeGroupConfigurations {
+			workerConf := clusterSpec.NutanixMachineConfig(worker.MachineGroupRef.Name)
+			workerConfs[worker.MachineGroupRef.Name] = workerConf.Spec
+		}
+		dcConf := clusterSpec.NutanixDatacenter
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
+		creds := GetCredsFromEnv()
+		builder := NewNutanixTemplateBuilder(&dcConf.Spec, &machineConf.Spec, &machineConf.Spec, workerConfs, creds, time.Now)
+		assert.NotNil(t, builder)
+
+		buildSpec := test.NewFullClusterSpec(t, tc.Input)
+
+		cpSpec, err := builder.GenerateCAPISpecControlPlane(buildSpec)
+		assert.NoError(t, err)
+		assert.NotNil(t, cpSpec)
+		test.AssertContentToFile(t, string(cpSpec), tc.OutputCP)
+
+		workloadTemplateNames := map[string]string{
+			"eksa-unit-test": "eksa-unit-test",
+		}
+		kubeadmconfigTemplateNames := map[string]string{
+			"eksa-unit-test": "eksa-unit-test",
+		}
+		workerSpec, err := builder.GenerateCAPISpecWorkers(buildSpec, workloadTemplateNames, kubeadmconfigTemplateNames)
+		assert.NoError(t, err)
+		assert.NotNil(t, workerSpec)
+		test.AssertContentToFile(t, string(workerSpec), tc.OutputMD)
+	}
+}
+
 func minimalNutanixConfigSpec(t *testing.T) (*anywherev1.NutanixDatacenterConfig, *anywherev1.NutanixMachineConfig, map[string]anywherev1.NutanixMachineConfigSpec) {
 	dcConf := &anywherev1.NutanixDatacenterConfig{}
 	err := yaml.Unmarshal([]byte(nutanixDatacenterConfigSpec), dcConf)