[WIP] Added logic for provisioning On-premises Cassandra cluster

.env.tmpl

@@ -1,4 +1,6 @@
 # For Instaclustr API
 USERNAME=""
 APIKEY=""
-HOSTNAME=""
+HOSTNAME=""
+ICADMIN_USERNAME=""
+ICADMIN_APIKEY=""

Makefile

@@ -6,6 +6,10 @@ OPERATOR_NAMESPACE ?= instaclustr-operator
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.24.2
 
+KUBEVIRT_TAG?=v1.57.0
+KUBEVIRT_VERSION?=v1.0.0
+ARCH?=linux-amd64
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -196,3 +200,21 @@ dev-build: docker-build kind-load deploy ## builds docker-image, loads it to kin
 kind-load: ## loads given image to kind cluster
 	kind load docker-image ${IMG}
 
+.PHONY: deploy-kubevirt
+deploy-kubevirt: ## Deploy kubevirt operator and custom resources
+	kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml
+	kubectl create -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml
+	kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-operator.yaml
+	kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-cr.yaml
+
+.PHONY: undeploy-kubevirt
+undeploy-kubevirt: ## Delete kubevirt operator and custom resources
+	kubectl delete apiservices v1.subresources.kubevirt.io --ignore-not-found
+	kubectl delete mutatingwebhookconfigurations virt-api-mutator --ignore-not-found
+	kubectl delete validatingwebhookconfigurations virt-operator-validator --ignore-not-found
+	kubectl delete validatingwebhookconfigurations virt-api-validator --ignore-not-found
+	kubectl delete -n kubevirt kubevirt kubevirt --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-cr.yaml --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/containerized-data-importer/releases/download/${KUBEVIRT_TAG}/cdi-operator.yaml --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml --ignore-not-found
+	kubectl delete -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-operator.yaml --ignore-not-found

apis/clusterresources/v1beta1/maintenanceevents_types.go

@@ -51,9 +51,9 @@ type MaintenanceEventStatus struct {
 }
 
 type ClusteredMaintenanceEventStatus struct {
-	InProgress []*MaintenanceEventStatus `json:"inProgress"`
-	Past       []*MaintenanceEventStatus `json:"past"`
-	Upcoming   []*MaintenanceEventStatus `json:"upcoming"`
+	InProgress []*MaintenanceEventStatus `json:"inProgress,omitempty"`
+	Past       []*MaintenanceEventStatus `json:"past,omitempty"`
+	Upcoming   []*MaintenanceEventStatus `json:"upcoming,omitempty"`
 }
 
 //+kubebuilder:object:root=true

apis/clusterresources/v1beta1/postgresqluser_types.go

@@ -40,11 +40,6 @@ type ClusterInfo struct {
 	Event                       string         `json:"event,omitempty"`
 }
 
-type NamespacedName struct {
-	Namespace string `json:"namespace"`
-	Name      string `json:"name"`
-}
-
 //+kubebuilder:object:root=true
 //+kubebuilder:subresource:status
 

apis/clusterresources/v1beta1/structs.go

@@ -57,3 +57,8 @@ type SecretReference struct {
 	Namespace string `json:"namespace"`
 	Name      string `json:"name"`
 }
+
+type NamespacedName struct {
+	Namespace string `json:"namespace"`
+	Name      string `json:"name"`
+}

apis/clusters/v1beta1/cassandra_types.go

@@ -55,7 +55,8 @@ type CassandraRestoreFrom struct {
 
 // CassandraSpec defines the desired state of Cassandra
 type CassandraSpec struct {
-	RestoreFrom         *CassandraRestoreFrom `json:"restoreFrom,omitempty"`
+	RestoreFrom         *CassandraRestoreFrom    `json:"restoreFrom,omitempty"`
+	OnPremisesSpec      *CassandraOnPremisesSpec `json:"onPremisesSpec,omitempty"`
 	Cluster             `json:",inline"`
 	DataCentres         []*CassandraDataCentre `json:"dataCentres,omitempty"`
 	LuceneEnabled       bool                   `json:"luceneEnabled,omitempty"`
@@ -67,6 +68,18 @@ type CassandraSpec struct {
 	ResizeSettings []*ResizeSettings `json:"resizeSettings,omitempty"`
 }
 
+type CassandraOnPremisesSpec struct {
+	StorageClassName              string          `json:"storageClassName"`
+	OSDiskSize                    string          `json:"osDiskSize"`
+	DataDiskSize                  string          `json:"dataDiskSize"`
+	SSHGatewayCPU                 int64           `json:"sshGatewayCPU,omitempty"`
+	SSHGatewayMemory              string          `json:"sshGatewayMemory,omitempty"`
+	NodeCPU                       int64           `json:"nodeCPU"`
+	NodeMemory                    string          `json:"nodeMemory"`
+	OSImageURL                    string          `json:"osImageURL"`
+	CloudInitScriptNamespacedName *NamespacedName `json:"cloudInitScriptNamespacedName"`
+}
+
 // CassandraStatus defines the observed state of Cassandra
 type CassandraStatus struct {
 	ClusterStatus `json:",inline"`
@@ -141,7 +154,7 @@ func (c *Cassandra) NewBackupSpec(startTimestamp int) *clusterresourcesv1beta1.C
 	return &clusterresourcesv1beta1.ClusterBackup{
 		TypeMeta: ctrl.TypeMeta{
 			Kind:       models.ClusterBackupKind,
-			APIVersion: models.ClusterresourcesV1beta1APIVersion,
+			APIVersion: models.ClusterResourcesV1beta1APIVersion,
 		},
 		ObjectMeta: ctrl.ObjectMeta{
 			Name:        models.SnapshotUploadPrefix + c.Status.ID + "-" + strconv.Itoa(startTimestamp),

apis/clusters/v1beta1/cassandra_webhook.go

@@ -19,6 +19,7 @@ package v1beta1
 import (
 	"context"
 	"fmt"
+	"regexp"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -87,6 +88,37 @@ func (cv *cassandraValidator) ValidateCreate(ctx context.Context, obj runtime.Ob
 		return err
 	}
 
+	if c.Spec.OnPremisesSpec != nil {
+		osDiskSizeMatched, err := regexp.Match(models.StorageRegExp, []byte(c.Spec.OnPremisesSpec.OSDiskSize))
+		if !osDiskSizeMatched || err != nil {
+			return fmt.Errorf("disk size field for node OS must fit pattern: %s",
+				models.StorageRegExp)
+		}
+
+		dataDiskSizeMatched, err := regexp.Match(models.StorageRegExp, []byte(c.Spec.OnPremisesSpec.DataDiskSize))
+		if !dataDiskSizeMatched || err != nil {
+			return fmt.Errorf("disk size field for storring cluster data must fit pattern: %s",
+				models.StorageRegExp)
+		}
+
+		nodeMemoryMatched, err := regexp.Match(models.MemoryRegExp, []byte(c.Spec.OnPremisesSpec.DataDiskSize))
+		if !nodeMemoryMatched || err != nil {
+			return fmt.Errorf("node memory field must fit pattern: %s",
+				models.MemoryRegExp)
+		}
+
+		if c.Spec.PrivateNetworkCluster {
+			if c.Spec.OnPremisesSpec.SSHGatewayCPU == 0 || c.Spec.OnPremisesSpec.SSHGatewayMemory == "" {
+				return fmt.Errorf("fields SSHGatewayCPU and SSHGatewayMemory must not be empty")
+			}
+			sshGatewayMemoryMatched, err := regexp.Match(models.MemoryRegExp, []byte(c.Spec.OnPremisesSpec.DataDiskSize))
+			if !sshGatewayMemoryMatched || err != nil {
+				return fmt.Errorf("ssh gateway memory field must fit pattern: %s",
+					models.MemoryRegExp)
+			}
+		}
+	}
+
 	if len(c.Spec.Spark) > 1 {
 		return fmt.Errorf("spark should not have more than 1 item")
 	}
@@ -113,10 +145,22 @@ func (cv *cassandraValidator) ValidateCreate(ctx context.Context, obj runtime.Ob
 		return fmt.Errorf("data centres field is empty")
 	}
 
+	//TODO: add support of multiple DCs for OnPrem clusters
+	if len(c.Spec.DataCentres) > 1 && c.Spec.OnPremisesSpec != nil {
+		return fmt.Errorf("on-premises cluster can be provisioned with only one data centre")
+	}
+
 	for _, dc := range c.Spec.DataCentres {
-		err := dc.DataCentre.ValidateCreation()
-		if err != nil {
-			return err
+		if c.Spec.OnPremisesSpec != nil {
+			err := dc.DataCentre.ValidateOnPremisesCreation()
+			if err != nil {
+				return err
+			}
+		} else {
+			err := dc.DataCentre.ValidateCreation()
+			if err != nil {
+				return err
+			}
 		}
 
 		if !c.Spec.PrivateNetworkCluster && dc.PrivateIPBroadcastForDiscovery {

apis/clusters/v1beta1/opensearch_types.go

@@ -539,7 +539,7 @@ func (os *OpenSearch) NewBackupSpec(startTimestamp int) *clusterresourcesv1beta1
 	return &clusterresourcesv1beta1.ClusterBackup{
 		TypeMeta: ctrl.TypeMeta{
 			Kind:       models.ClusterBackupKind,
-			APIVersion: models.ClusterresourcesV1beta1APIVersion,
+			APIVersion: models.ClusterResourcesV1beta1APIVersion,
 		},
 		ObjectMeta: ctrl.ObjectMeta{
 			Name:        models.SnapshotUploadPrefix + os.Status.ID + "-" + strconv.Itoa(startTimestamp),

apis/clusters/v1beta1/postgresql_types.go

@@ -150,7 +150,7 @@ func (pg *PostgreSQL) NewBackupSpec(startTimestamp int) *clusterresourcesv1beta1
 	return &clusterresourcesv1beta1.ClusterBackup{
 		TypeMeta: ctrl.TypeMeta{
 			Kind:       models.ClusterBackupKind,
-			APIVersion: models.ClusterresourcesV1beta1APIVersion,
+			APIVersion: models.ClusterResourcesV1beta1APIVersion,
 		},
 		ObjectMeta: ctrl.ObjectMeta{
 			Name:        models.PgBackupPrefix + pg.Status.ID + "-" + strconv.Itoa(startTimestamp),

apis/clusters/v1beta1/redis_types.go

@@ -132,7 +132,7 @@ func (r *Redis) NewBackupSpec(startTimestamp int) *clusterresourcesv1beta1.Clust
 	return &clusterresourcesv1beta1.ClusterBackup{
 		TypeMeta: ctrl.TypeMeta{
 			Kind:       models.ClusterBackupKind,
-			APIVersion: models.ClusterresourcesV1beta1APIVersion,
+			APIVersion: models.ClusterResourcesV1beta1APIVersion,
 		},
 		ObjectMeta: ctrl.ObjectMeta{
 			Name:        models.SnapshotUploadPrefix + r.Status.ID + "-" + strconv.Itoa(startTimestamp),

apis/clusters/v1beta1/structs.go

@@ -143,6 +143,61 @@ type privateLinkStatus struct {
 	EndPointServiceName string `json:"endPointServiceName,omitempty"`
 }
 
+type NamespacedName struct {
+	Namespace string `json:"namespace"`
+	Name      string `json:"name"`
+}
+
+type Gateway struct {
+	ID                   string `json:"id,omitempty"`
+	ClusterDataCentre    string `json:"clusterDataCentre,omitempty"`
+	ClusterID            string `json:"clusterId,omitempty"`
+	PublicAddress        string `json:"publicAddress,omitempty"`
+	PrivateAddress       string `json:"privateAddress,omitempty"`
+	NatID                string `json:"natId,omitempty"`
+	NatPublicAddress     string `json:"natPublicAddress,omitempty"`
+	NatPrivateAddress    string `json:"natPrivateAddress,omitempty"`
+	NodeAgentVersion     string `json:"nodeAgentVersion,omitempty"`
+	SSHMarkedForDeletion string `json:"sshMarkedForDeletion,omitempty"`
+	SSHReplaces          string `json:"sshReplaces,omitempty"`
+	NatMarkedForDeletion string `json:"natMarkedForDeletion,omitempty"`
+	Rack                 string `json:"rack,omitempty"`
+	RackID               string `json:"rackId,omitempty"`
+	SSHAWSID             string `json:"sshAWSId,omitempty"`
+}
+
+type OnPremiseNode struct {
+	ID                         string `json:"id,omitempty"`
+	ClusterDataCentre          string `json:"clusterDataCentre,omitempty"`
+	AccountID                  string `json:"accountId,omitempty"`
+	Status                     string `json:"status,omitempty"`
+	PublicAddress              string `json:"publicAddress,omitempty"`
+	PrivateAddress             string `json:"privateAddress,omitempty"`
+	Provider                   string `json:"provider,omitempty"`
+	Size                       string `json:"size,omitempty"`
+	DeferredReason             string `json:"deferredReason,omitempty"`
+	MarkedForDeletion          string `json:"markedForDeletion,omitempty"`
+	NodeAgentStartDate         string `json:"nodeAgentStartDate,omitempty"`
+	ChargifyDateLastBilled     string `json:"chargifyDateLastBilled,omitempty"`
+	LastOSUpdate               string `json:"lastOSUpdate,omitempty"`
+	Replaces                   string `json:"replaces,omitempty"`
+	Rack                       string `json:"rack,omitempty"`
+	RackID                     string `json:"rackId,omitempty"`
+	DataCentre                 string `json:"dataCentre,omitempty"`
+	ForceStart                 bool   `json:"forceStart,omitempty"`
+	BundleStartEnabled         bool   `json:"bundleStartEnabled,omitempty"`
+	ClusterID                  string `json:"clusterId,omitempty"`
+	EphemeralStorageDiskCount  int    `json:"ephemeralStorageDiskCount,omitempty"`
+	PersistentStorageDiskCount int    `json:"persistentStorageDiskCount,omitempty"`
+	CacheDiskQuota             int    `json:"cacheDiskQuota,omitempty"`
+	FailureReason              string `json:"failureReason,omitempty"`
+	NodeAgentVersion           string `json:"nodeAgentVersion,omitempty"`
+	OSVersionID                string `json:"osVersionId,omitempty"`
+	OSBuildID                  string `json:"osBuildId,omitempty"`
+	DiskQuota                  int    `json:"diskQuota,omitempty"`
+	InstanceStore              bool   `json:"instanceStore,omitempty"`
+}
+
 type PrivateLinkStatuses []*privateLinkStatus
 
 func (p1 PrivateLinkStatuses) Equal(p2 PrivateLinkStatuses) bool {

apis/clusters/v1beta1/validation.go

@@ -47,6 +47,25 @@ func (c *Cluster) ValidateCreation() error {
 	return nil
 }
 
+func (dc *DataCentre) ValidateOnPremisesCreation() error {
+	if dc.CloudProvider != models.ONPREMISES {
+		return fmt.Errorf("cloud provider %s is unavailable for data centre: %s, available value: %s",
+			dc.CloudProvider, dc.Name, models.ONPREMISES)
+	}
+
+	if dc.Region != models.CLIENTDC {
+		return fmt.Errorf("region %s is unavailable for data centre: %s, available value: %s",
+			dc.Region, dc.Name, models.CLIENTDC)
+	}
+
+	if !validation.Contains(dc.NodeSize, models.CassandraOnPremNodes) {
+		return fmt.Errorf("on-premises node size: %s is unavailable, available sizes: %v",
+			dc.Region, models.CassandraOnPremNodes)
+	}
+
+	return nil
+}
+
 func (dc *DataCentre) ValidateCreation() error {
 	if !validation.Contains(dc.CloudProvider, models.CloudProviders) {
 		return fmt.Errorf("cloud provider %s is unavailable for data centre: %s, available values: %v",