diff --git a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml index 057f6647c7..d67cf97022 100644 --- a/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml +++ b/config/crd/bases/controlplane.cluster.x-k8s.io_rosacontrolplanes.yaml @@ -45,6 +45,17 @@ spec: type: object spec: properties: + autoscaling: + description: Autoscaling specifies auto scaling behaviour for the + MachinePools. + properties: + maxReplicas: + minimum: 1 + type: integer + minReplicas: + minimum: 1 + type: integer + type: object availabilityZones: description: AWS AvailabilityZones of the worker nodes should match the AvailabilityZones of the Subnets. @@ -113,10 +124,41 @@ spec: description: 'TODO: these are to satisfy ocm sdk. Explore how to drop them.' type: string - machineCIDR: - description: Block of IP addresses used by OpenShift while installing - the cluster, for example "10.0.0.0/16". + instanceType: + description: The instance type to use, for example `r5.xlarge`. Instance + type ref; https://aws.amazon.com/ec2/instance-types/ type: string + network: + description: Network config for the ROSA HCP cluster. + properties: + hostPrefix: + default: 23 + description: Network host prefix which is defaulted to `23` if + not specified. + type: integer + machineCIDR: + description: IP addresses block used by OpenShift while installing + the cluster, for example "10.0.0.0/16". + format: cidr + type: string + networkType: + default: OVNKubernetes + description: The CNI network type default is OVNKubernetes. + enum: + - OVNKubernetes + - Other + type: string + podCIDR: + description: IP address block from which to assign pod IP addresses, + for example `10.128.0.0/14`. + format: cidr + type: string + serviceCIDR: + description: IP address block from which to assign service IP + addresses, for example `172.30.0.0/16`. + format: cidr + type: string + type: object oidcID: description: The ID of the OpenID Connect Provider. type: string @@ -307,7 +349,6 @@ spec: required: - availabilityZones - installerRoleARN - - machineCIDR - oidcID - region - rolesRef diff --git a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go index dfb8688b66..65ecd9279b 100644 --- a/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go +++ b/controlplane/rosa/api/v1beta2/rosacontrolplane_types.go @@ -21,6 +21,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -43,9 +44,6 @@ type RosaControlPlaneSpec struct { //nolint: maligned // should match the AvailabilityZones of the Subnets. AvailabilityZones []string `json:"availabilityZones"` - // Block of IP addresses used by OpenShift while installing the cluster, for example "10.0.0.0/16". - MachineCIDR *string `json:"machineCIDR"` - // The AWS Region the cluster lives in. Region *string `json:"region"` @@ -90,6 +88,44 @@ type RosaControlPlaneSpec struct { //nolint: maligned // IdentityRef is a reference to an identity to be used when reconciling the managed control plane. // If no identity is specified, the default identity for this controller will be used. IdentityRef *infrav1.AWSIdentityReference `json:"identityRef,omitempty"` + + // Network config for the ROSA HCP cluster. + Network *NetworkSpec `json:"network,omitempty"` + + // The instance type to use, for example `r5.xlarge`. Instance type ref; https://aws.amazon.com/ec2/instance-types/ + // +optional + InstanceType string `json:"instanceType,omitempty"` + + // Autoscaling specifies auto scaling behaviour for the MachinePools. + // +optional + Autoscaling *expinfrav1.RosaMachinePoolAutoScaling `json:"autoscaling,omitempty"` +} + +// NetworkSpec for ROSA-HCP. +type NetworkSpec struct { + // IP addresses block used by OpenShift while installing the cluster, for example "10.0.0.0/16". + // +kubebuilder:validation:Format=cidr + MachineCIDR string `json:"machineCIDR,omitempty"` + + // IP address block from which to assign pod IP addresses, for example `10.128.0.0/14`. + // +kubebuilder:validation:Format=cidr + // +optional + PodCIDR string `json:"podCIDR,omitempty"` + + // IP address block from which to assign service IP addresses, for example `172.30.0.0/16`. + // +kubebuilder:validation:Format=cidr + // +optional + ServiceCIDR string `json:"serviceCIDR,omitempty"` + + // Network host prefix which is defaulted to `23` if not specified. + // +kubebuilder:default=23 + // +optional + HostPrefix int `json:"hostPrefix,omitempty"` + + // The CNI network type default is OVNKubernetes. + // +kubebuilder:validation:Enum=OVNKubernetes;Other + // +kubebuilder:default=OVNKubernetes + NetworkType string `json:"networkType,omitempty"` } // AWSRolesRef contains references to various AWS IAM roles required for operators to make calls against the AWS API. diff --git a/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go b/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go index bfb980ac14..41dac04354 100644 --- a/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go +++ b/controlplane/rosa/api/v1beta2/zz_generated.deepcopy.go @@ -24,6 +24,7 @@ import ( "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" apiv1beta2 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + expapiv1beta2 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2" "sigs.k8s.io/cluster-api/api/v1beta1" ) @@ -42,6 +43,21 @@ func (in *AWSRolesRef) DeepCopy() *AWSRolesRef { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkSpec. +func (in *NetworkSpec) DeepCopy() *NetworkSpec { + if in == nil { + return nil + } + out := new(NetworkSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ROSAControlPlane) DeepCopyInto(out *ROSAControlPlane) { *out = *in @@ -114,11 +130,6 @@ func (in *RosaControlPlaneSpec) DeepCopyInto(out *RosaControlPlaneSpec) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.MachineCIDR != nil { - in, out := &in.MachineCIDR, &out.MachineCIDR - *out = new(string) - **out = **in - } if in.Region != nil { in, out := &in.Region, &out.Region *out = new(string) @@ -156,6 +167,16 @@ func (in *RosaControlPlaneSpec) DeepCopyInto(out *RosaControlPlaneSpec) { *out = new(apiv1beta2.AWSIdentityReference) **out = **in } + if in.Network != nil { + in, out := &in.Network, &out.Network + *out = new(NetworkSpec) + **out = **in + } + if in.Autoscaling != nil { + in, out := &in.Autoscaling, &out.Autoscaling + *out = new(expapiv1beta2.RosaMachinePoolAutoScaling) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RosaControlPlaneSpec. diff --git a/controlplane/rosa/controllers/rosacontrolplane_controller.go b/controlplane/rosa/controllers/rosacontrolplane_controller.go index f7ad335d1f..69e6e31c8a 100644 --- a/controlplane/rosa/controllers/rosacontrolplane_controller.go +++ b/controlplane/rosa/controllers/rosacontrolplane_controller.go @@ -262,12 +262,6 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc return ctrl.Result{RequeueAfter: time.Second * 60}, nil } - _, machineCIDR, err := net.ParseCIDR(*rosaScope.ControlPlane.Spec.MachineCIDR) - if err != nil { - // TODO: expose in status, exit reconciliation - rosaScope.Error(err, "rosacontrolplane.spec.machineCIDR invalid") - } - billingAccount := *rosaScope.Identity.Account if rosaScope.ControlPlane.Spec.BillingAccount != "" { billingAccount = rosaScope.ControlPlane.Spec.BillingAccount @@ -283,59 +277,18 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc Expiration: time.Now().Add(1 * time.Hour), DisableWorkloadMonitoring: ptr.To(true), DefaultIngress: ocm.NewDefaultIngressSpec(), // n.b. this is a no-op when it's set to the default value + ComputeMachineType: rosaScope.ControlPlane.Spec.InstanceType, SubnetIds: rosaScope.ControlPlane.Spec.Subnets, AvailabilityZones: rosaScope.ControlPlane.Spec.AvailabilityZones, - NetworkType: "OVNKubernetes", - MachineCIDR: *machineCIDR, + NetworkType: rosaScope.ControlPlane.Spec.Network.NetworkType, IsSTS: true, RoleARN: *rosaScope.ControlPlane.Spec.InstallerRoleARN, SupportRoleARN: *rosaScope.ControlPlane.Spec.SupportRoleARN, - OperatorIAMRoles: []ocm.OperatorIAMRole{ - { - Name: "cloud-credentials", - Namespace: "openshift-ingress-operator", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.IngressARN, - }, - { - Name: "installer-cloud-credentials", - Namespace: "openshift-image-registry", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.ImageRegistryARN, - }, - { - Name: "ebs-cloud-credentials", - Namespace: "openshift-cluster-csi-drivers", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.StorageARN, - }, - { - Name: "cloud-credentials", - Namespace: "openshift-cloud-network-config-controller", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.NetworkARN, - }, - { - Name: "kube-controller-manager", - Namespace: "kube-system", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.KubeCloudControllerARN, - }, - { - Name: "kms-provider", - Namespace: "kube-system", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.KMSProviderARN, - }, - { - Name: "control-plane-operator", - Namespace: "kube-system", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.ControlPlaneOperatorARN, - }, - { - Name: "capa-controller-manager", - Namespace: "kube-system", - RoleARN: rosaScope.ControlPlane.Spec.RolesRef.NodePoolManagementARN, - }, - }, - WorkerRoleARN: *rosaScope.ControlPlane.Spec.WorkerRoleARN, - OidcConfigId: *rosaScope.ControlPlane.Spec.OIDCID, - Mode: "auto", + OperatorIAMRoles: getOperatorIAMRole(*rosaScope.ControlPlane), + WorkerRoleARN: *rosaScope.ControlPlane.Spec.WorkerRoleARN, + OidcConfigId: *rosaScope.ControlPlane.Spec.OIDCID, + Mode: "auto", Hypershift: ocm.Hypershift{ Enabled: true, }, @@ -343,6 +296,43 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc AWSCreator: creator, } + _, machineCIDR, err := net.ParseCIDR(rosaScope.ControlPlane.Spec.Network.MachineCIDR) + if err == nil { + spec.MachineCIDR = *machineCIDR + } else { + // TODO: expose in status + rosaScope.Error(err, "rosacontrolplane.spec.network.machineCIDR invalid", rosaScope.ControlPlane.Spec.Network.MachineCIDR) + return ctrl.Result{}, nil + } + + if rosaScope.ControlPlane.Spec.Network.PodCIDR != "" { + _, podCIDR, err := net.ParseCIDR(rosaScope.ControlPlane.Spec.Network.PodCIDR) + if err == nil { + spec.PodCIDR = *podCIDR + } else { + // TODO: expose in status. + rosaScope.Error(err, "rosacontrolplane.spec.network.podCIDR invalid", rosaScope.ControlPlane.Spec.Network.PodCIDR) + return ctrl.Result{}, nil + } + } + + if rosaScope.ControlPlane.Spec.Network.ServiceCIDR != "" { + _, serviceCIDR, err := net.ParseCIDR(rosaScope.ControlPlane.Spec.Network.ServiceCIDR) + if err == nil { + spec.ServiceCIDR = *serviceCIDR + } else { + // TODO: expose in status. + rosaScope.Error(err, "rosacontrolplane.spec.network.serviceCIDR invalid", rosaScope.ControlPlane.Spec.Network.ServiceCIDR) + return ctrl.Result{}, nil + } + } + + // Set autoscale replica + if rosaScope.ControlPlane.Spec.Autoscaling != nil { + spec.MaxReplicas = rosaScope.ControlPlane.Spec.Autoscaling.MaxReplicas + spec.MinReplicas = rosaScope.ControlPlane.Spec.Autoscaling.MinReplicas + } + cluster, err = ocmClient.CreateCluster(spec) if err != nil { // TODO: need to expose in status, as likely the spec is invalid @@ -356,6 +346,51 @@ func (r *ROSAControlPlaneReconciler) reconcileNormal(ctx context.Context, rosaSc return ctrl.Result{}, nil } +func getOperatorIAMRole(rosaControlPlane rosacontrolplanev1.ROSAControlPlane) []ocm.OperatorIAMRole { + return []ocm.OperatorIAMRole{ + { + Name: "cloud-credentials", + Namespace: "openshift-ingress-operator", + RoleARN: rosaControlPlane.Spec.RolesRef.IngressARN, + }, + { + Name: "installer-cloud-credentials", + Namespace: "openshift-image-registry", + RoleARN: rosaControlPlane.Spec.RolesRef.ImageRegistryARN, + }, + { + Name: "ebs-cloud-credentials", + Namespace: "openshift-cluster-csi-drivers", + RoleARN: rosaControlPlane.Spec.RolesRef.StorageARN, + }, + { + Name: "cloud-credentials", + Namespace: "openshift-cloud-network-config-controller", + RoleARN: rosaControlPlane.Spec.RolesRef.NetworkARN, + }, + { + Name: "kube-controller-manager", + Namespace: "kube-system", + RoleARN: rosaControlPlane.Spec.RolesRef.KubeCloudControllerARN, + }, + { + Name: "kms-provider", + Namespace: "kube-system", + RoleARN: rosaControlPlane.Spec.RolesRef.KMSProviderARN, + }, + { + Name: "control-plane-operator", + Namespace: "kube-system", + RoleARN: rosaControlPlane.Spec.RolesRef.ControlPlaneOperatorARN, + }, + { + Name: "capa-controller-manager", + Namespace: "kube-system", + RoleARN: rosaControlPlane.Spec.RolesRef.NodePoolManagementARN, + }, + } +} + func (r *ROSAControlPlaneReconciler) reconcileDelete(ctx context.Context, rosaScope *scope.ROSAControlPlaneScope) (res ctrl.Result, reterr error) { rosaScope.Info("Reconciling ROSAControlPlane delete") diff --git a/templates/cluster-template-rosa-machinepool.yaml b/templates/cluster-template-rosa-machinepool.yaml index ce5d04a728..c86266fe5d 100644 --- a/templates/cluster-template-rosa-machinepool.yaml +++ b/templates/cluster-template-rosa-machinepool.yaml @@ -32,7 +32,8 @@ spec: region: "${AWS_REGION}" accountID: "${AWS_ACCOUNT_ID}" creatorARN: "${AWS_CREATOR_ARN}" - machineCIDR: "10.0.0.0/16" + network: + machineCIDR: "10.0.0.0/16" rolesRef: ingressARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-ingress-operator-cloud-credentials" imageRegistryARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-image-registry-installer-cloud-credentials" diff --git a/templates/cluster-template-rosa.yaml b/templates/cluster-template-rosa.yaml index 8fd60b9a5f..763575134d 100644 --- a/templates/cluster-template-rosa.yaml +++ b/templates/cluster-template-rosa.yaml @@ -32,7 +32,8 @@ spec: region: "${AWS_REGION}" accountID: "${AWS_ACCOUNT_ID}" creatorARN: "${AWS_CREATOR_ARN}" - machineCIDR: "10.0.0.0/16" + network: + machineCIDR: "10.0.0.0/16" rolesRef: ingressARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-ingress-operator-cloud-credentials" imageRegistryARN: "arn:aws:iam::${AWS_ACCOUNT_ID}:role/${OPERATOR_ROLES_PREFIX}-openshift-image-registry-installer-cloud-credentials"