Add Nutanix CCM ignore node IPs list

config/crd/bases/anywhere.eks.amazonaws.com_nutanixdatacenterconfigs.yaml

@@ -42,6 +42,13 @@ spec:
                   bundle for users that configured their Prism Central with certificates
                   from non-publicly trusted CAs
                 type: string
+              ccmExcludeNodeIPs:
+                description: CcmExcludeIPs is the optional list of IP addresses that
+                  should be excluded from the CCM IP pool for nodes. List should be
+                  valid IP addresses and IP address ranges.
+                items:
+                  type: string
+                type: array
               credentialRef:
                 description: CredentialRef is the reference to the secret name that
                   contains the credentials for the Nutanix Prism Central. The namespace

pkg/api/v1alpha1/nutanixdatacenterconfig_types.go

@@ -47,6 +47,11 @@ type NutanixDatacenterConfigSpec struct {
 	// FailureDomains is the optional list of failure domains for the Nutanix Datacenter.
 	// +optional
 	FailureDomains []NutanixDatacenterFailureDomain `json:"failureDomains,omitempty"`
+
+	// CcmExcludeIPs is the optional list of IP addresses that should be excluded from the CCM IP pool for nodes.
+	// List should be valid IP addresses and IP address ranges.
+	// +optional
+	CcmExcludeNodeIPs []string `json:"ccmExcludeNodeIPs,omitempty"`
 }
 
 // NutanixDatacenterFailureDomain defines the failure domain for the Nutanix Datacenter.

pkg/clustermanager/cluster_manager.go

@@ -100,6 +100,7 @@ type CAPIClient interface {
 	GetWorkloadKubeconfig(ctx context.Context, clusterName string, cluster *types.Cluster) ([]byte, error)
 }
 
+// AwsIamAuth performs operations with AWS IAM.
 type AwsIamAuth interface {
 	CreateAndInstallAWSIAMAuthCASecret(ctx context.Context, managementCluster *types.Cluster, workloadClusterName string) error
 	InstallAWSIAMAuth(ctx context.Context, management, workload *types.Cluster, spec *cluster.Spec) error

pkg/providers/nutanix/config/cp-template.yaml

@@ -647,7 +647,8 @@ data:
           "enableCustomLabeling": false,
           "topologyDiscovery": {
             "type": "Prism"
-          }
+          },
+          "ignoredNodeIPs": [{{ range $i, $ip := .ccmIgnoredNodeIPs }}{{ if $i }}, {{ end }}"{{ $ip }}"{{ end }}]
         }
     ---
     apiVersion: rbac.authorization.k8s.io/v1

pkg/providers/nutanix/template.go

@@ -4,6 +4,9 @@
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"log"
+	"net"
+	"strings"
 
 	"sigs.k8s.io/yaml"
 
@@ -176,9 +179,12 @@
 
 	failureDomains := generateNutanixFailureDomains(datacenterSpec.FailureDomains)
 
+	ccmIgnoredNodeIPs := generateCcmIgnoredNodeIPsList(clusterSpec)
+
 	values := map[string]interface{}{
 		"auditPolicy":                  auditPolicy,
 		"apiServerExtraArgs":           apiServerExtraArgs.ToPartialYaml(),
+		"ccmIgnoredNodeIPs":            ccmIgnoredNodeIPs,
 		"cloudProviderImage":           versionsBundle.Nutanix.CloudProvider.VersionedImage(),
 		"clusterName":                  clusterSpec.Cluster.Name,
 		"controlPlaneEndpointIp":       clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host,
@@ -568,3 +574,118 @@
 	}
 	return failureDomains
 }
+
+func incrementIP(ip net.IP) {
+	for i := len(ip) - 1; i >= 0; i-- {
+		ip[i]++
+		if ip[i] > 0 {
+			break
+		}
+	}
+}
+
+func compareIP(ip1, ip2 net.IP) (int, error) {
+	if len(ip1) != len(ip2) {
+		return -1, fmt.Errorf("IP addresses are not the same protocol")
+	}
+
+	for i := 0; i < len(ip1); i++ {
+		if ip1[i] < ip2[i] {
+			return -1, nil
+		}
+		if ip1[i] > ip2[i] {
+			return 1, nil
+		}
+	}
+
+	return 0, nil
+}
+
+func addCIDRToIgnoredNodeIPsList(cidr string, result []string) []string {
+	ip, ipNet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		// log error and continue
+		log.Printf("error parsing CIDR %s: %v", cidr, err)
+		return result
+	}
+
+	// Add all ip addresses in the range to the list
+	for ip := ip.Mask(ipNet.Mask); ipNet.Contains(ip); incrementIP(ip) {
+		if ip != nil {
+			result = append(result, ip.String())
+		}
+	}
+
+	return result
+}
+
+func addIPRangeToIgnoredNodeIPsList(ipRangeStr string, result []string) []string {
+	// Parse the range
+	ipRange := strings.Split(ipRangeStr, "-")
+	if len(ipRange) != 2 {
+		// log error and return
+		log.Printf("error parsing range %s: expected 2 values, got %d", ipRangeStr, len(ipRange))
+		return result
+	}
+
+	// Parse the start and end of the range
+	start := net.ParseIP(strings.TrimSpace(ipRange[0]))
+	end := net.ParseIP(strings.TrimSpace(ipRange[1]))
+	if start == nil || end == nil {
+		// log error and return
+		log.Printf("error parsing range %s: invalid IP address", ipRangeStr)
+		return result
+	}
+
+	cmp, err := compareIP(start, end)
+	if err != nil {
+		// log error and return
+		log.Printf("error comparing IP addresses %s and %s: %v", start.String(), end.String(), err)
+		return result
+	}
+
+	if cmp >= 0 {
+		// swap start and end if start is greater than end
+		start, end = end, start
+	}
+
+	// Add all ip addresses in the range to the list
+	for ip := start; !ip.Equal(end); incrementIP(ip) {
+		result = append(result, ip.String())
+	}
+
+	result = append(result, end.String())
+
+	return result
+}
+
+func addIPAddressToIgnoredNodeIPsList(ipAddrStr string, result []string) []string {
+	ip := net.ParseIP(ipAddrStr)
+	if ip == nil {
+		// log error and return
+		log.Printf("error parsing IP address %s", ipAddrStr)
+		return result
+	}
+
+	result = append(result, ip.String())
+	return result
+}
+
+func generateCcmIgnoredNodeIPsList(clusterSpec *cluster.Spec) []string {
+	// Add the kube-vip IP address to the list
+	result := []string{clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host}
+
+	// Add the IP addresses, IP ranges and CIDRs to the list from the NutanixDatacenter spec CcmExcludeNodeIPs
+	for _, IPAddrOrRange := range clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs {
+		addrOrRange := strings.TrimSpace(IPAddrOrRange)
+		if strings.Contains(addrOrRange, "/") {
+			result = addCIDRToIgnoredNodeIPsList(addrOrRange, result)
+		} else if strings.Contains(addrOrRange, "-") {
+			result = addIPRangeToIgnoredNodeIPsList(addrOrRange, result)
+		} else {
+			result = addIPAddressToIgnoredNodeIPsList(addrOrRange, result)
+		}
+	}
+
+	return result
+}

pkg/providers/nutanix/template_test.go

@@ -848,6 +848,92 @@ func TestTemplateBuilderGPUs(t *testing.T) {
 	}
 }
 
+func TestTemplateBuilderCcmExcludeNodeIPs(t *testing.T) {
+	for _, tc := range []struct {
+		Input    string
+		Output   string
+		ChangeFn func(clusterSpec *cluster.Spec) *cluster.Spec
+	}{
+		{
+			Input:  "testdata/eksa-cluster-ccm-exclude-node-ips.yaml",
+			Output: "testdata/expected_cluster_ccm_exclude_node_ips.yaml",
+			ChangeFn: func(clusterSpec *cluster.Spec) *cluster.Spec {
+				excludeNodeIPs := []string{
+					"127.100.200.101",
+					"10.10.10.10-10.10.10.13",
+					"10.123.0.0/29",
+				}
+				clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs = excludeNodeIPs
+
+				return clusterSpec
+			},
+		},
+		{
+			Input:  "testdata/eksa-cluster-ccm-exclude-node-ips.yaml",
+			Output: "testdata/expected_cluster_ccm_exclude_node_ips.yaml",
+			ChangeFn: func(clusterSpec *cluster.Spec) *cluster.Spec {
+				excludeNodeIPs := []string{
+					"127.100.200.101",
+					"10.10.10.10-10.10.10.13",
+					"10.123.0.0/29",
+					"10.10.10.20-10.10.10.30-10.10.20.30",
+				}
+				clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs = excludeNodeIPs
+
+				return clusterSpec
+			},
+		},
+		{
+			Input:  "testdata/eksa-cluster-ccm-exclude-node-ips.yaml",
+			Output: "testdata/expected_cluster_ccm_exclude_node_ips.yaml",
+			ChangeFn: func(clusterSpec *cluster.Spec) *cluster.Spec {
+				excludeNodeIPs := []string{
+					"127.100.200.101",
+					"10.10.10.10-10.10.10.13",
+					"10.123.0.0/29",
+					"244.244.1",
+				}
+				clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs = excludeNodeIPs
+
+				return clusterSpec
+			},
+		},
+		{
+			Input:  "testdata/eksa-cluster-ccm-exclude-node-ips.yaml",
+			Output: "testdata/expected_cluster_ccm_exclude_node_ips.yaml",
+			ChangeFn: func(clusterSpec *cluster.Spec) *cluster.Spec {
+				excludeNodeIPs := []string{
+					"127.100.200.101",
+					"10.10.10.10-10.10.10.13",
+					"10.123.0.0/29",
+					"10.21.0.5/55",
+				}
+				clusterSpec.NutanixDatacenter.Spec.CcmExcludeNodeIPs = excludeNodeIPs
+
+				return clusterSpec
+			},
+		},
+	} {
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+
+		machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
+		creds := GetCredsFromEnv()
+
+		clusterSpec = tc.ChangeFn(clusterSpec)
+
+		bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,
+			map[string]anywherev1.NutanixMachineConfigSpec{}, creds, time.Now)
+
+		cpSpec, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
+		assert.NoError(t, err)
+		assert.NotNil(t, cpSpec)
+		test.AssertContentToFile(t, string(cpSpec), tc.Output)
+	}
+}
+
 func minimalNutanixConfigSpec(t *testing.T) (*anywherev1.NutanixDatacenterConfig, *anywherev1.NutanixMachineConfig, map[string]anywherev1.NutanixMachineConfigSpec) {
 	dcConf := &anywherev1.NutanixDatacenterConfig{}
 	err := yaml.Unmarshal([]byte(nutanixDatacenterConfigSpec), dcConf)

pkg/providers/nutanix/testdata/cluster_api_server_cert_san_domain_name.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     certSANs: ["foo.bar"]
     machineGroupRef:
       name: test

pkg/providers/nutanix/testdata/cluster_api_server_cert_san_ip.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     certSANs: ["11.11.11.11"]
     machineGroupRef:
       name: test

pkg/providers/nutanix/testdata/cluster_nutanix_etcd_encryption.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_etcd_encryption_1_29.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_failure_domains.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_invalid_trust_bundle.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_trust_bundle.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_upgrade_strategy_cp.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_upgrade_strategy_md.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips.yaml

@@ -0,0 +1,15 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30