Add Nutanix CCM ignore node IPs list

pkg/clustermanager/cluster_manager.go

@@ -100,6 +100,7 @@ type CAPIClient interface {
 	GetWorkloadKubeconfig(ctx context.Context, clusterName string, cluster *types.Cluster) ([]byte, error)
 }
 
+// AwsIamAuth interface on AWS IAM.
 type AwsIamAuth interface {
 	CreateAndInstallAWSIAMAuthCASecret(ctx context.Context, managementCluster *types.Cluster, workloadClusterName string) error
 	InstallAWSIAMAuth(ctx context.Context, management, workload *types.Cluster, spec *cluster.Spec) error

pkg/providers/nutanix/config/cp-template.yaml

@@ -647,7 +647,8 @@ data:
           "enableCustomLabeling": false,
           "topologyDiscovery": {
             "type": "Prism"
-          }
+          },
+          "ignoredNodeIPs": [{{ range $i, $ip := .ccmIgnoredNodeIPs }}{{ if $i }}, {{ end }}"{{ $ip }}"{{ end }}]
         }
     ---
     apiVersion: rbac.authorization.k8s.io/v1

pkg/providers/nutanix/template.go

@@ -176,9 +176,12 @@ func buildTemplateMapCP(
 
 	failureDomains := generateNutanixFailureDomains(datacenterSpec.FailureDomains)
 
+	ccmIgnoredNodeIPs := generateCcmIgnoredNodeIPsList(clusterSpec)
+
 	values := map[string]interface{}{
 		"auditPolicy":                  auditPolicy,
 		"apiServerExtraArgs":           apiServerExtraArgs.ToPartialYaml(),
+		"ccmIgnoredNodeIPs":            ccmIgnoredNodeIPs,
 		"cloudProviderImage":           versionsBundle.Nutanix.CloudProvider.VersionedImage(),
 		"clusterName":                  clusterSpec.Cluster.Name,
 		"controlPlaneEndpointIp":       clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host,
@@ -519,3 +522,9 @@ func generateNutanixFailureDomains(eksNutanixFailureDomains []v1alpha1.NutanixDa
 	}
 	return failureDomains
 }
+
+func generateCcmIgnoredNodeIPsList(clusterSpec *cluster.Spec) []string {
+	ignoredIPs := []string{clusterSpec.Cluster.Spec.ControlPlaneConfiguration.Endpoint.Host}
+
+	return ignoredIPs
+}

pkg/providers/nutanix/template_test.go

@@ -726,6 +726,34 @@ func TestTemplateBuilderFailureDomains(t *testing.T) {
 	}
 }
 
+func TestTemplateBuilderCcmExcludeNodeIPs(t *testing.T) {
+	for _, tc := range []struct {
+		Input  string
+		Output string
+	}{
+		{
+			Input:  "testdata/eksa-cluster-ccm-exclude-node-ips.yaml",
+			Output: "testdata/expected_cluster_ccm_exclude_node_ips.yaml",
+		},
+	} {
+		clusterSpec := test.NewFullClusterSpec(t, tc.Input)
+
+		machineCfg := clusterSpec.NutanixMachineConfig(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.MachineGroupRef.Name)
+
+		t.Setenv(constants.EksaNutanixUsernameKey, "admin")
+		t.Setenv(constants.EksaNutanixPasswordKey, "password")
+		creds := GetCredsFromEnv()
+
+		bldr := NewNutanixTemplateBuilder(&clusterSpec.NutanixDatacenter.Spec, &machineCfg.Spec, nil,
+			map[string]anywherev1.NutanixMachineConfigSpec{}, creds, time.Now)
+
+		cpSpec, err := bldr.GenerateCAPISpecControlPlane(clusterSpec)
+		assert.NoError(t, err)
+		assert.NotNil(t, cpSpec)
+		test.AssertContentToFile(t, string(cpSpec), tc.Output)
+	}
+}
+
 func minimalNutanixConfigSpec(t *testing.T) (*anywherev1.NutanixDatacenterConfig, *anywherev1.NutanixMachineConfig, map[string]anywherev1.NutanixMachineConfigSpec) {
 	dcConf := &anywherev1.NutanixDatacenterConfig{}
 	err := yaml.Unmarshal([]byte(nutanixDatacenterConfigSpec), dcConf)

pkg/providers/nutanix/testdata/cluster_api_server_cert_san_domain_name.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     certSANs: ["foo.bar"]
     machineGroupRef:
       name: test

pkg/providers/nutanix/testdata/cluster_api_server_cert_san_ip.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     certSANs: ["11.11.11.11"]
     machineGroupRef:
       name: test

pkg/providers/nutanix/testdata/cluster_nutanix_etcd_encryption.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_etcd_encryption_1_29.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_failure_domains.yaml

@@ -9,7 +9,7 @@ spec:
     name: test
     count: 1
     endpoint:
-      host: test
+      host: 10.199.199.1
     machineGroupRef:
       name: test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_invalid_trust_bundle.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_trust_bundle.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_upgrade_strategy_cp.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/cluster_nutanix_with_upgrade_strategy_md.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips.yaml

@@ -0,0 +1,15 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_cidr.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 10.100.0.0//16

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 244.244.01

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip_range1.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 10.100.0.10-10.100.10.10-10.100.20.30

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip_range2.yaml

@@ -0,0 +1,16 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 192.179.1.1-10.1.1.1

pkg/providers/nutanix/testdata/datacenterConfig_with_ccm_exclude_node_ips_invalid_ip_range3.yaml

@@ -0,0 +1,17 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIPs:
+    - 10.0.0.1
+    - 10.0.0.0/24
+    - 10.0.0.10-10.0.0.30
+    - 10.0.10.0-::1
+

pkg/providers/nutanix/testdata/eksa-cluster-additional-categories.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/eksa-cluster-autoscaler.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/eksa-cluster-ccm-exclude-node-ips.yaml

@@ -0,0 +1,73 @@
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: Cluster
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  kubernetesVersion: "1.19"
+  controlPlaneConfiguration:
+    name: eksa-unit-test
+    count: 3
+    endpoint:
+      host: 10.199.199.1
+    machineGroupRef:
+      name: eksa-unit-test
+      kind: NutanixMachineConfig
+  workerNodeGroupConfigurations:
+    - count: 4
+      name: eksa-unit-test
+      machineGroupRef:
+        name: eksa-unit-test
+        kind: NutanixMachineConfig
+  datacenterRef:
+    kind: NutanixDatacenterConfig
+    name: eksa-unit-test
+  clusterNetwork:
+    cni: "cilium"
+    pods:
+      cidrBlocks:
+        - 192.168.0.0/16
+    services:
+      cidrBlocks:
+        - 10.96.0.0/12
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixDatacenterConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  endpoint: "prism.nutanix.com"
+  port: 9440
+  credentialRef:
+    kind: Secret
+    name: "nutanix-credentials"
+  ccmExcludeNodeIps:
+    - "127.100.200.101"
+    - "10.10.10.10-10.10.10.13"
+    - "10.123.0.0/29"
+---
+apiVersion: anywhere.eks.amazonaws.com/v1alpha1
+kind: NutanixMachineConfig
+metadata:
+  name: eksa-unit-test
+  namespace: default
+spec:
+  vcpusPerSocket: 1
+  vcpuSockets: 4
+  memorySize: 8Gi
+  image:
+    type: "name"
+    name: "prism-image-1-19"
+  cluster:
+    type: "name"
+    name: "prism-cluster"
+  subnet:
+    type: "name"
+    name: "prism-subnet"
+  systemDiskSize: 40Gi
+  osFamily: "ubuntu"
+  users:
+    - name: "mySshUsername"
+      sshAuthorizedKeys:
+        - "mySshAuthorizedKey"

pkg/providers/nutanix/testdata/eksa-cluster-external-etcd-k8s-1-20.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test-cp
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/eksa-cluster-external-etcd-with-optional.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test-cp
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig

pkg/providers/nutanix/testdata/eksa-cluster-external-etcd.yaml

@@ -9,7 +9,7 @@ spec:
     name: eksa-unit-test-cp
     count: 3
     endpoint:
-      host: test-ip
+      host: 10.199.199.1
     machineGroupRef:
       name: eksa-unit-test
       kind: NutanixMachineConfig