Merge pull request #4850 from muraee/rosa-cleanup-api

✨ ROSA: Cleanup
kubernetes-sigs · Mar 12, 2024 · 52469b2 · 52469b2
2 parents 5aa9d22 + d60bc63
commit 52469b2
Show file tree

Hide file tree

Showing 8 changed files with 514 additions and 181 deletions.
diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml
@@ -53,20 +53,10 @@ spec:
                 description: AdditionalTags are user-defined tags to be added on the
                   AWS resources associated with the control plane.
                 type: object
-              autoscaling:
-                description: Autoscaling specifies auto scaling behaviour for the
-                  MachinePools.
-                properties:
-                  maxReplicas:
-                    minimum: 1
-                    type: integer
-                  minReplicas:
-                    minimum: 1
-                    type: integer
-                type: object
               availabilityZones:
-                description: AWS AvailabilityZones of the worker nodes should match
-                  the AvailabilityZones of the Subnets.
+                description: AvailabilityZones describe AWS AvailabilityZones of the
+                  worker nodes. should match the AvailabilityZones of the provided
+                  Subnets. a machinepool will be created for each availabilityZone.
                 items:
                   type: string
                 type: array
@@ -108,6 +98,33 @@ spec:
                     type: string
                 type: object
                 x-kubernetes-map-type: atomic
+              defaultMachinePoolSpec:
+                description: "DefaultMachinePoolSpec defines the configuration for
+                  the default machinepool(s) provisioned as part of the cluster creation.
+                  One MachinePool will be created with this configuration per AvailabilityZone.
+                  Those default machinepools are required for openshift cluster operators
+                  to work properly. As these machinepool not created using ROSAMachinePool
+                  CR, they will not be visible/managed by ROSA CAPI provider. `rosa
+                  list machinepools -c <rosaClusterName>` can be used to view those
+                  machinepools. \n This field will be removed in the future once the
+                  current limitation is resolved."
+                properties:
+                  autoscaling:
+                    description: Autoscaling specifies auto scaling behaviour for
+                      this MachinePool.
+                    properties:
+                      maxReplicas:
+                        minimum: 1
+                        type: integer
+                      minReplicas:
+                        minimum: 1
+                        type: integer
+                    type: object
+                  instanceType:
+                    description: The instance type to use, for example `r5.xlarge`.
+                      Instance type ref; https://aws.amazon.com/ec2/instance-types/
+                    type: string
+                type: object
               domainPrefix:
                 description: DomainPrefix is an optional prefix added to the cluster's
                   domain name. It will be used when generating a sub-domain for the
@@ -155,12 +172,8 @@ spec:
                 - name
                 type: object
               installerRoleARN:
-                description: 'TODO: these are to satisfy ocm sdk. Explore how to drop
-                  them.'
-                type: string
-              instanceType:
-                description: The instance type to use, for example `r5.xlarge`. Instance
-                  type ref; https://aws.amazon.com/ec2/instance-types/
+                description: InstallerRoleARN is an AWS IAM role that OpenShift Cluster
+                  Manager will assume to create the cluster..
                 type: string
               network:
                 description: Network config for the ROSA HCP cluster.
@@ -371,11 +384,15 @@ spec:
                   type: string
                 type: array
               supportRoleARN:
+                description: SupportRoleARN is an AWS IAM role used by Red Hat SREs
+                  to enable access to the cluster account in order to provide support.
                 type: string
               version:
                 description: OpenShift semantic version, for example "4.14.5".
                 type: string
               workerRoleARN:
+                description: WorkerRoleARN is an AWS IAM role that will be attached
+                  to worker instances.
                 type: string
             required:
             - availabilityZones

diff --git a/controlplane/rosa/api/v1beta2/conditions_consts.go b/controlplane/rosa/api/v1beta2/conditions_consts.go
@@ -31,6 +31,9 @@ const (
 	// ROSAControlPlaneReconciliationFailedReason used to report failures while reconciling ROSAControlPlane.
 	ROSAControlPlaneReconciliationFailedReason = "ReconciliationFailed"
 
+	// ROSAControlPlaneDeletionFailedReason used to report failures while deleting ROSAControlPlane.
+	ROSAControlPlaneDeletionFailedReason = "DeletionFailed"
+
 	// ROSAControlPlaneInvalidConfigurationReason used to report invalid user input.
 	ROSAControlPlaneInvalidConfigurationReason = "InvalidConfiguration"
 )
diff --git a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go
@@ -66,8 +66,9 @@ type RosaControlPlaneSpec struct { //nolint: maligned
 	// SubnetIDs should come in pairs; two per availability zone, one private and one public.
 	Subnets []string `json:"subnets"`
 
-	// AWS AvailabilityZones of the worker nodes
-	// should match the AvailabilityZones of the Subnets.
+	// AvailabilityZones describe AWS AvailabilityZones of the worker nodes.
+	// should match the AvailabilityZones of the provided Subnets.
+	// a machinepool will be created for each availabilityZone.
 	AvailabilityZones []string `json:"availabilityZones"`
 
 	// The AWS Region the cluster lives in.
@@ -82,10 +83,13 @@ type RosaControlPlaneSpec struct { //nolint: maligned
 	// The ID of the OpenID Connect Provider.
 	OIDCID string `json:"oidcID"`
 
-	// TODO: these are to satisfy ocm sdk. Explore how to drop them.
-	InstallerRoleARN *string `json:"installerRoleARN"`
-	SupportRoleARN   *string `json:"supportRoleARN"`
-	WorkerRoleARN    *string `json:"workerRoleARN"`
+	// InstallerRoleARN is an AWS IAM role that OpenShift Cluster Manager will assume to create the cluster..
+	InstallerRoleARN string `json:"installerRoleARN"`
+	// SupportRoleARN is an AWS IAM role used by Red Hat SREs to enable
+	// access to the cluster account in order to provide support.
+	SupportRoleARN string `json:"supportRoleARN"`
+	// WorkerRoleARN is an AWS IAM role that will be attached to worker instances.
+	WorkerRoleARN string `json:"workerRoleARN"`
 
 	// BillingAccount is an optional AWS account to use for billing the subscription fees for ROSA clusters.
 	// The cost of running each ROSA cluster will be billed to the infrastructure account in which the cluster
@@ -98,18 +102,16 @@ type RosaControlPlaneSpec struct { //nolint: maligned
 	// +optional
 	BillingAccount string `json:"billingAccount,omitempty"`
 
-	// CredentialsSecretRef references a secret with necessary credentials to connect to the OCM API.
-	// The secret should contain the following data keys:
-	// - ocmToken: eyJhbGciOiJIUzI1NiIsI....
-	// - ocmApiUrl: Optional, defaults to 'https://api.openshift.com'
-	// +optional
-	CredentialsSecretRef *corev1.LocalObjectReference `json:"credentialsSecretRef,omitempty"`
-
-	// IdentityRef is a reference to an identity to be used when reconciling the managed control plane.
-	// If no identity is specified, the default identity for this controller will be used.
+	// DefaultMachinePoolSpec defines the configuration for the default machinepool(s) provisioned as part of the cluster creation.
+	// One MachinePool will be created with this configuration per AvailabilityZone. Those default machinepools are required for openshift cluster operators
+	// to work properly.
+	// As these machinepool not created using ROSAMachinePool CR, they will not be visible/managed by ROSA CAPI provider.
+	// `rosa list machinepools -c <rosaClusterName>` can be used to view those machinepools.
+	//
+	// This field will be removed in the future once the current limitation is resolved.
 	//
 	// +optional
-	IdentityRef *infrav1.AWSIdentityReference `json:"identityRef,omitempty"`
+	DefaultMachinePoolSpec DefaultMachinePoolSpec `json:"defaultMachinePoolSpec,omitempty"`
 
 	// Network config for the ROSA HCP cluster.
 	// +optional
@@ -123,14 +125,6 @@ type RosaControlPlaneSpec struct { //nolint: maligned
 	// +optional
 	EndpointAccess RosaEndpointAccessType `json:"endpointAccess,omitempty"`
 
-	// The instance type to use, for example `r5.xlarge`. Instance type ref; https://aws.amazon.com/ec2/instance-types/
-	// +optional
-	InstanceType string `json:"instanceType,omitempty"`
-
-	// Autoscaling specifies auto scaling behaviour for the MachinePools.
-	// +optional
-	Autoscaling *expinfrav1.RosaMachinePoolAutoScaling `json:"autoscaling,omitempty"`
-
 	// AdditionalTags are user-defined tags to be added on the AWS resources associated with the control plane.
 	// +optional
 	AdditionalTags infrav1.Tags `json:"additionalTags,omitempty"`
@@ -140,6 +134,19 @@ type RosaControlPlaneSpec struct { //nolint: maligned
 	// +optional
 	EtcdEncryptionKMSArn string `json:"etcdEncryptionKMSArn,omitempty"`
 
+	// CredentialsSecretRef references a secret with necessary credentials to connect to the OCM API.
+	// The secret should contain the following data keys:
+	// - ocmToken: eyJhbGciOiJIUzI1NiIsI....
+	// - ocmApiUrl: Optional, defaults to 'https://api.openshift.com'
+	// +optional
+	CredentialsSecretRef *corev1.LocalObjectReference `json:"credentialsSecretRef,omitempty"`
+
+	// IdentityRef is a reference to an identity to be used when reconciling the managed control plane.
+	// If no identity is specified, the default identity for this controller will be used.
+	//
+	// +optional
+	IdentityRef *infrav1.AWSIdentityReference `json:"identityRef,omitempty"`
+
 	// ControlPlaneEndpoint represents the endpoint used to communicate with the control plane.
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
@@ -174,6 +181,17 @@ type NetworkSpec struct {
 	NetworkType string `json:"networkType,omitempty"`
 }
 
+// DefaultMachinePoolSpec defines the configuration for the required worker nodes provisioned as part of the cluster creation.
+type DefaultMachinePoolSpec struct {
+	// The instance type to use, for example `r5.xlarge`. Instance type ref; https://aws.amazon.com/ec2/instance-types/
+	// +optional
+	InstanceType string `json:"instanceType,omitempty"`
+
+	// Autoscaling specifies auto scaling behaviour for this MachinePool.
+	// +optional
+	Autoscaling *expinfrav1.RosaMachinePoolAutoScaling `json:"autoscaling,omitempty"`
+}
+
 // AWSRolesRef contains references to various AWS IAM roles required for operators to make calls against the AWS API.
 type AWSRolesRef struct {
 	// The referenced role must have a trust relationship that allows it to be assumed via web identity.

diff --git a/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go b/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go
diff --git a/controlplane/rosa/controllers/rosacontrolplane_controller.go b/controlplane/rosa/controllers/rosacontrolplane_controller.go
@@ -50,6 +50,7 @@ import (
 
 	rosacontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/rosa/api/v1beta2"
 	expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/annotations"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/scope"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/rosa"
@@ -66,6 +67,9 @@ const (
 	rosaControlPlaneKind = "ROSAControlPlane"
 	// ROSAControlPlaneFinalizer allows the controller to clean up resources on delete.
 	ROSAControlPlaneFinalizer = "rosacontrolplane.controlplane.cluster.x-k8s.io"
+
+	// ROSAControlPlaneForceDeleteAnnotation annotation can be set to force the deletion of ROSAControlPlane bypassing any deletion validations/errors.
+	ROSAControlPlaneForceDeleteAnnotation = "controlplane.cluster.x-k8s.io/rosacontrolplane-force-delete"
 )
 
 // ROSAControlPlaneReconciler reconciles a ROSAControlPlane object.
@@ -283,20 +287,20 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc
 		ChannelGroup:              ocm.DefaultChannelGroup,
 		DisableWorkloadMonitoring: ptr.To(true),
 		DefaultIngress:            ocm.NewDefaultIngressSpec(), // n.b. this is a no-op when it's set to the default value
-		ComputeMachineType:        rosaScope.ControlPlane.Spec.InstanceType,
+		ComputeMachineType:        rosaScope.ControlPlane.Spec.DefaultMachinePoolSpec.InstanceType,
+		AvailabilityZones:         rosaScope.ControlPlane.Spec.AvailabilityZones,
 		Tags:                      rosaScope.ControlPlane.Spec.AdditionalTags,
 		EtcdEncryption:            rosaScope.ControlPlane.Spec.EtcdEncryptionKMSArn != "",
 		EtcdEncryptionKMSArn:      rosaScope.ControlPlane.Spec.EtcdEncryptionKMSArn,
 
-		SubnetIds:         rosaScope.ControlPlane.Spec.Subnets,
-		AvailabilityZones: rosaScope.ControlPlane.Spec.AvailabilityZones,
-		IsSTS:             true,
-		RoleARN:           *rosaScope.ControlPlane.Spec.InstallerRoleARN,
-		SupportRoleARN:    *rosaScope.ControlPlane.Spec.SupportRoleARN,
-		WorkerRoleARN:     *rosaScope.ControlPlane.Spec.WorkerRoleARN,
-		OperatorIAMRoles:  operatorIAMRoles(rosaScope.ControlPlane.Spec.RolesRef),
-		OidcConfigId:      rosaScope.ControlPlane.Spec.OIDCID,
-		Mode:              "auto",
+		SubnetIds:        rosaScope.ControlPlane.Spec.Subnets,
+		IsSTS:            true,
+		RoleARN:          rosaScope.ControlPlane.Spec.InstallerRoleARN,
+		SupportRoleARN:   rosaScope.ControlPlane.Spec.SupportRoleARN,
+		WorkerRoleARN:    rosaScope.ControlPlane.Spec.WorkerRoleARN,
+		OperatorIAMRoles: operatorIAMRoles(rosaScope.ControlPlane.Spec.RolesRef),
+		OidcConfigId:     rosaScope.ControlPlane.Spec.OIDCID,
+		Mode:             "auto",
 		Hypershift: ocm.Hypershift{
 			Enabled: true,
 		},
@@ -338,10 +342,10 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc
 		ocmClusterSpec.NetworkType = networkSpec.NetworkType
 	}
 
-	// Set autoscale replica
-	if rosaScope.ControlPlane.Spec.Autoscaling != nil {
-		ocmClusterSpec.MaxReplicas = rosaScope.ControlPlane.Spec.Autoscaling.MaxReplicas
-		ocmClusterSpec.MinReplicas = rosaScope.ControlPlane.Spec.Autoscaling.MinReplicas
+	// Set cluster compute autoscaling replicas
+	if computeAutoscaling := rosaScope.ControlPlane.Spec.DefaultMachinePoolSpec.Autoscaling; computeAutoscaling != nil {
+		ocmClusterSpec.MaxReplicas = computeAutoscaling.MaxReplicas
+		ocmClusterSpec.MinReplicas = computeAutoscaling.MinReplicas
 	}
 
 	cluster, err = ocmClient.CreateCluster(ocmClusterSpec)
@@ -429,12 +433,29 @@ func (r *ROSAControlPlaneReconciler) reconcileDelete(ctx context.Context, rosaSc
 		return ctrl.Result{}, nil
 	}
 
+	bestEffort := false
+	if value, found := annotations.Get(rosaScope.ControlPlane, ROSAControlPlaneForceDeleteAnnotation); found && value != "false" {
+		bestEffort = true
+	}
+
 	if cluster.Status().State() != cmv1.ClusterStateUninstalling {
-		if _, err := ocmClient.DeleteCluster(cluster.ID(), true, creator); err != nil {
+		if _, err := ocmClient.DeleteCluster(cluster.ID(), bestEffort, creator); err != nil {
+			conditions.MarkFalse(rosaScope.ControlPlane,
+				rosacontrolplanev1.ROSAControlPlaneReadyCondition,
+				rosacontrolplanev1.ROSAControlPlaneDeletionFailedReason,
+				clusterv1.ConditionSeverityError,
+				"failed to delete ROSAControlPlane: %s; if the error can't be resolved, set '%s' annotation to force the deletion",
+				err.Error(),
+				ROSAControlPlaneForceDeleteAnnotation)
 			return ctrl.Result{}, err
 		}
 	}
 
+	conditions.MarkFalse(rosaScope.ControlPlane,
+		rosacontrolplanev1.ROSAControlPlaneReadyCondition,
+		string(cluster.Status().State()),
+		clusterv1.ConditionSeverityInfo,
+		"deleting")
 	rosaScope.ControlPlane.Status.Ready = false
 	rosaScope.Info("waiting for cluster to be deleted")
 	// Requeue to remove the finalizer when the cluster is fully deleted.