Add e2e tests for apiServerExtraArgs create and upgrade cluster

aws · Mar 15, 2024 · 2709e4a · 2709e4a
1 parent 4ff0842
commit 2709e4a
Show file tree

Hide file tree

Showing 16 changed files with 162 additions and 14 deletions.
diff --git a/config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml b/config/crd/bases/anywhere.eks.amazonaws.com_clusters.yaml
@@ -154,7 +154,7 @@ spec:
                   apiServerExtraArgs:
                     additionalProperties:
                       type: string
-                    description: ApiServerExtraArgs defines the flags to configure
+                    description: APIServerExtraArgs defines the flags to configure
                       for the API server.
                     type: object
                   certSans:

diff --git a/config/manifest/eksa-components.yaml b/config/manifest/eksa-components.yaml
@@ -3857,7 +3857,7 @@ spec:
                   apiServerExtraArgs:
                     additionalProperties:
                       type: string
-                    description: ApiServerExtraArgs defines the flags to configure
+                    description: APIServerExtraArgs defines the flags to configure
                       for the API server.
                     type: object
                   certSans:

diff --git a/internal/pkg/api/cluster.go b/internal/pkg/api/cluster.go
@@ -126,6 +126,26 @@ func WithControlPlaneLabel(key string, val string) ClusterFiller {
 	}
 }
 
+// WithControlPlaneAPIServerExtraArgs adds the APIServerExtraArgs to the cluster spec.
+func WithControlPlaneAPIServerExtraArgs() ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		if c.Spec.ControlPlaneConfiguration.APIServerExtraArgs == nil {
+			c.Spec.ControlPlaneConfiguration.APIServerExtraArgs = map[string]string{}
+		}
+		issuerURL := "https://" + c.Spec.ControlPlaneConfiguration.Endpoint.Host
+		c.Spec.ControlPlaneConfiguration.APIServerExtraArgs["service-account-jwks-uri"] = issuerURL + "/openid/v1/jwks"
+	}
+}
+
+// RemoveAllAPIServerExtraArgs removes all the API server flags from the cluster spec.
+func RemoveAllAPIServerExtraArgs() ClusterFiller {
+	return func(c *anywherev1.Cluster) {
+		for k := range c.Spec.ControlPlaneConfiguration.APIServerExtraArgs {
+			delete(c.Spec.ControlPlaneConfiguration.APIServerExtraArgs, k)
+		}
+	}
+}
+
 // WithPodCidr sets an explicit pod CIDR, overriding the provider's default.
 func WithPodCidr(podCidr string) ClusterFiller {
 	return func(c *anywherev1.Cluster) {

diff --git a/pkg/api/v1alpha1/cluster_types.go b/pkg/api/v1alpha1/cluster_types.go
@@ -307,8 +307,8 @@ type ControlPlaneConfiguration struct {
 	CertSANs []string `json:"certSans,omitempty"`
 	// MachineHealthCheck is a control-plane level override for the timeouts and maxUnhealthy specified in the top-level MHC configuration. If not configured, the defaults in the top-level MHC configuration are used.
 	MachineHealthCheck *MachineHealthCheck `json:"machineHealthCheck,omitempty"`
-	// ApiServerExtraArgs defines the flags to configure for the API server.
-	ApiServerExtraArgs map[string]string `json:"apiServerExtraArgs,omitempty"`
+	// APIServerExtraArgs defines the flags to configure for the API server.
+	APIServerExtraArgs map[string]string `json:"apiServerExtraArgs,omitempty"`
 }
 
 // MachineHealthCheck allows to configure timeouts for machine health checks. Machine Health Checks are responsible for remediating unhealthy Machines.
@@ -365,7 +365,7 @@ func (n *ControlPlaneConfiguration) Equal(o *ControlPlaneConfiguration) bool {
 	}
 	return n.Count == o.Count && n.MachineGroupRef.Equal(o.MachineGroupRef) &&
 		TaintsSliceEqual(n.Taints, o.Taints) && MapEqual(n.Labels, o.Labels) &&
-		SliceEqual(n.CertSANs, o.CertSANs) && MapEqual(n.ApiServerExtraArgs, o.ApiServerExtraArgs)
+		SliceEqual(n.CertSANs, o.CertSANs) && MapEqual(n.APIServerExtraArgs, o.APIServerExtraArgs)
 }
 
 type Endpoint struct {

diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/pkg/clusterapi/extraargs.go b/pkg/clusterapi/extraargs.go
@@ -53,7 +53,8 @@ func EtcdEncryptionExtraArgs(config *[]v1alpha1.EtcdEncryption) ExtraArgs {
 	return args
 }
 
-func ApiServerExtraArgs(apiServerExtraArgs map[string]string) ExtraArgs {
+// APIServerExtraArgs takes a map of API Server extra args and returns the relevant API server extra args if it's not nil or empty.
+func APIServerExtraArgs(apiServerExtraArgs map[string]string) ExtraArgs {
 	args := ExtraArgs{}
 	for k, v := range apiServerExtraArgs {
 		args.AddIfNotEmpty(k, v)

diff --git a/pkg/clusterapi/extraargs_test.go b/pkg/clusterapi/extraargs_test.go
@@ -205,7 +205,7 @@ func TestAPIServerExtraArgs(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.testName, func(t *testing.T) {
-			if got := clusterapi.ApiServerExtraArgs(tt.apiServerExtraArgs); !reflect.DeepEqual(got, tt.want) {
+			if got := clusterapi.APIServerExtraArgs(tt.apiServerExtraArgs); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("APIServerExtraArgs() = %v, want %v", got, tt.want)
 			}
 		})

diff --git a/pkg/providers/cloudstack/template.go b/pkg/providers/cloudstack/template.go
@@ -123,7 +123,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
 		Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)).
-		Append(clusterapi.ApiServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.ApiServerExtraArgs)).
+		Append(clusterapi.APIServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.APIServerExtraArgs)).
 		Append(sharedExtraArgs)
 
 	controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().

diff --git a/pkg/providers/docker/docker.go b/pkg/providers/docker/docker.go
@@ -295,7 +295,7 @@ func buildTemplateMapCP(clusterSpec *cluster.Spec) (map[string]interface{}, erro
 	apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig).
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
-		Append(clusterapi.ApiServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.ApiServerExtraArgs)).
+		Append(clusterapi.APIServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.APIServerExtraArgs)).
 		Append(sharedExtraArgs)
 	controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
 		Append(clusterapi.NodeCIDRMaskExtraArgs(&clusterSpec.Cluster.Spec.ClusterNetwork))

diff --git a/pkg/providers/nutanix/template.go b/pkg/providers/nutanix/template.go
@@ -163,7 +163,7 @@ func buildTemplateMapCP(
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
 		Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)).
-		Append(clusterapi.ApiServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.ApiServerExtraArgs))
+		Append(clusterapi.APIServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.APIServerExtraArgs))
 	kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
 		Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf)).
 		Append(clusterapi.ControlPlaneNodeLabelsExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration))

diff --git a/pkg/providers/tinkerbell/template.go b/pkg/providers/tinkerbell/template.go
@@ -400,7 +400,7 @@ func buildTemplateMapCP(
 	apiServerExtraArgs := clusterapi.OIDCToExtraArgs(clusterSpec.OIDCConfig).
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
-		Append(clusterapi.ApiServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.ApiServerExtraArgs))
+		Append(clusterapi.APIServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.APIServerExtraArgs))
 
 	kubeletExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
 		Append(clusterapi.ResolvConfExtraArgs(clusterSpec.Cluster.Spec.ClusterNetwork.DNS.ResolvConf)).

diff --git a/pkg/providers/vsphere/template.go b/pkg/providers/vsphere/template.go
@@ -145,7 +145,7 @@ func buildTemplateMapCP(
 		Append(clusterapi.AwsIamAuthExtraArgs(clusterSpec.AWSIamConfig)).
 		Append(clusterapi.PodIAMAuthExtraArgs(clusterSpec.Cluster.Spec.PodIAMConfig)).
 		Append(clusterapi.EtcdEncryptionExtraArgs(clusterSpec.Cluster.Spec.EtcdEncryption)).
-		Append(clusterapi.ApiServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.ApiServerExtraArgs)).
+		Append(clusterapi.APIServerExtraArgs(clusterSpec.Cluster.Spec.ControlPlaneConfiguration.APIServerExtraArgs)).
 		Append(sharedExtraArgs)
 	controllerManagerExtraArgs := clusterapi.SecureTlsCipherSuitesExtraArgs().
 		Append(clusterapi.NodeCIDRMaskExtraArgs(&clusterSpec.Cluster.Spec.ClusterNetwork))

diff --git a/test/e2e/api_server_extra_args.go b/test/e2e/api_server_extra_args.go
@@ -0,0 +1,19 @@
+//go:build e2e
+// +build e2e
+
+package e2e
+
+import (
+	"github.com/aws/eks-anywhere/test/framework"
+)
+
+func runAPIServerExtraArgsUpgradeFlow(test *framework.ClusterE2ETest, clusterOpts ...[]framework.ClusterE2ETestOpt) {
+	test.GenerateClusterConfig()
+	test.CreateCluster()
+	for _, opts := range clusterOpts {
+		test.UpgradeClusterWithNewConfig(opts)
+		test.ValidateClusterState()
+		test.StopIfFailed()
+	}
+	test.DeleteCluster()
+}
diff --git a/test/e2e/cloudstack_test.go b/test/e2e/cloudstack_test.go
@@ -17,6 +17,48 @@ import (
 	"github.com/aws/eks-anywhere/test/framework"
 )
 
+// APIServerExtraArgs
+func TestCloudStackKubernetes129RedHat8APIServerExtraArgsSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewCloudStack(t, framework.WithCloudStackRedhat129()),
+	).WithClusterConfig(
+		api.ClusterToConfigFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube129),
+			api.WithControlPlaneAPIServerExtraArgs(),
+		),
+	)
+	runSimpleFlowWithoutClusterConfigGeneration(test)
+}
+
+// TODO: Investigate why this test takes long time to pass with service-account-issuer flag
+func TestCloudStackKubernetes129Redhat8APIServerExtraArgsUpgradeFlow(t *testing.T) {
+	var addAPIServerExtraArgsclusterOpts []framework.ClusterE2ETestOpt
+	var removeAPIServerExtraArgsclusterOpts []framework.ClusterE2ETestOpt
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewCloudStack(t, framework.WithCloudStackRedhat129()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+	)
+	addAPIServerExtraArgsclusterOpts = append(
+		addAPIServerExtraArgsclusterOpts,
+		framework.WithClusterUpgrade(
+			api.WithControlPlaneAPIServerExtraArgs(),
+		),
+	)
+	removeAPIServerExtraArgsclusterOpts = append(
+		removeAPIServerExtraArgsclusterOpts,
+		framework.WithClusterUpgrade(
+			api.RemoveAllAPIServerExtraArgs(),
+		),
+	)
+	runAPIServerExtraArgsUpgradeFlow(
+		test,
+		addAPIServerExtraArgsclusterOpts,
+		removeAPIServerExtraArgsclusterOpts,
+	)
+}
+
 // AWS IAM Auth
 func TestCloudStackKubernetes125AWSIamAuth(t *testing.T) {
 	test := framework.NewClusterE2ETest(

diff --git a/test/e2e/vsphere_test.go b/test/e2e/vsphere_test.go
@@ -18,6 +18,48 @@ import (
 	"github.com/aws/eks-anywhere/test/framework"
 )
 
+// APIServerExtraArgs
+func TestVSphereKubernetes129BottlerocketAPIServerExtraArgsSimpleFlow(t *testing.T) {
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewVSphere(t, framework.WithBottleRocket129()),
+	).WithClusterConfig(
+		api.ClusterToConfigFiller(
+			api.WithKubernetesVersion(v1alpha1.Kube129),
+			api.WithControlPlaneAPIServerExtraArgs(),
+		),
+	)
+	runSimpleFlowWithoutClusterConfigGeneration(test)
+}
+
+// TODO: Investigate why this test takes long time to pass with service-account-issuer flag
+func TestVSphereKubernetes129BottlerocketAPIServerExtraArgsUpgradeFlow(t *testing.T) {
+	var addAPIServerExtraArgsclusterOpts []framework.ClusterE2ETestOpt
+	var removeAPIServerExtraArgsclusterOpts []framework.ClusterE2ETestOpt
+	test := framework.NewClusterE2ETest(
+		t,
+		framework.NewVSphere(t, framework.WithBottleRocket129()),
+		framework.WithClusterFiller(api.WithKubernetesVersion(v1alpha1.Kube129)),
+	)
+	addAPIServerExtraArgsclusterOpts = append(
+		addAPIServerExtraArgsclusterOpts,
+		framework.WithClusterUpgrade(
+			api.WithControlPlaneAPIServerExtraArgs(),
+		),
+	)
+	removeAPIServerExtraArgsclusterOpts = append(
+		removeAPIServerExtraArgsclusterOpts,
+		framework.WithClusterUpgrade(
+			api.RemoveAllAPIServerExtraArgs(),
+		),
+	)
+	runAPIServerExtraArgsUpgradeFlow(
+		test,
+		addAPIServerExtraArgsclusterOpts,
+		removeAPIServerExtraArgsclusterOpts,
+	)
+}
+
 // Autoimport
 func TestVSphereKubernetes125BottlerocketAutoimport(t *testing.T) {
 	provider := framework.NewVSphere(t,

diff --git a/test/framework/cluster/validations/cluster.go b/test/framework/cluster/validations/cluster.go
@@ -40,6 +40,9 @@ func ValidateClusterReady(ctx context.Context, vc clusterf.StateValidationConfig
 	if clus.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy != nil && clus.Spec.ControlPlaneConfiguration.UpgradeRolloutStrategy.Type == v1alpha1.InPlaceStrategyType {
 		return validateCAPIobjectsForInPlace(ctx, vc)
 	}
+	if clus.Spec.ControlPlaneConfiguration.APIServerExtraArgs != nil {
+		return validateKCPobjectForAPIServerExtraArgs(ctx, vc)
+	}
 	return nil
 }
 
@@ -360,3 +363,24 @@ func getWorkerNodeMachineSets(ctx context.Context, vc clusterf.StateValidationCo
 	}
 	return ms.Items, nil
 }
+
+func validateKCPobjectForAPIServerExtraArgs(ctx context.Context, vc clusterf.StateValidationConfig) error {
+	kcp, err := controller.GetKubeadmControlPlane(ctx, vc.ManagementClusterClient, vc.ClusterSpec.Cluster)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve kcp: %s", err)
+	}
+	if kcp == nil {
+		return errors.New("KubeadmControlPlane object not found")
+	}
+	apiServerExtraArgsKCP := kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.APIServer.ExtraArgs
+	apiServerExtraArgsSpec := vc.ClusterSpec.Cluster.Spec.ControlPlaneConfiguration.APIServerExtraArgs
+	if apiServerExtraArgsKCP == nil {
+		return fmt.Errorf("kcp object APIServerExtraArgs is nil expected: %v", apiServerExtraArgsSpec)
+	}
+	for k, v := range apiServerExtraArgsSpec {
+		if val, ok := apiServerExtraArgsKCP[k]; !ok || val != v {
+			return fmt.Errorf("kcp object does not have required APIServerExtraArgs expected: %v, actual: %v", apiServerExtraArgsSpec, apiServerExtraArgsKCP)
+		}
+	}
+	return nil
+}