From 944f92a70fd7ae1a6fb97587edb376f5695d1d44 Mon Sep 17 00:00:00 2001 From: Bohdan Siryk Date: Thu, 22 Feb 2024 15:28:50 +0200 Subject: [PATCH] cadence codebase refactor --- .secrets.baseline | 26 +- apis/clusters/v1beta1/cadence_types.go | 334 ++++++++++-------- apis/clusters/v1beta1/cadence_webhook.go | 139 ++++---- apis/clusters/v1beta1/cassandra_types.go | 3 +- apis/clusters/v1beta1/generic_spec.go | 5 +- apis/clusters/v1beta1/kafka_types.go | 3 +- apis/clusters/v1beta1/kafkaconnect_types.go | 2 +- apis/clusters/v1beta1/opensearch_types.go | 3 +- apis/clusters/v1beta1/redis_types.go | 3 +- apis/clusters/v1beta1/structs.go | 61 +--- .../clusters/v1beta1/zz_generated.deepcopy.go | 148 ++++---- .../clusters.instaclustr.com_cadences.yaml | 138 ++++++-- .../clusters.instaclustr.com_cassandras.yaml | 2 + config/samples/clusters_v1beta1_cadence.yaml | 5 +- .../samples/clusters_v1beta1_cassandra.yaml | 2 +- controllers/clusters/cadence_controller.go | 146 ++------ .../clusters/datatest/cadence_v1beta1.yaml | 2 +- .../datatest/cadence_v1beta1_packaged.yaml | 2 +- pkg/instaclustr/client.go | 10 +- pkg/instaclustr/interfaces.go | 2 +- pkg/instaclustr/mock/client.go | 2 +- pkg/models/apiv2.go | 18 + pkg/models/cadence_apiv2.go | 65 ++-- pkg/models/errors.go | 1 - 24 files changed, 561 insertions(+), 561 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index daa82b244..70a51c546 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -169,14 +169,14 @@ "filename": "apis/clusters/v1beta1/cadence_types.go", "hashed_secret": "a242f4a16b957f7ff99eb24e189e94d270d2348b", "is_verified": false, - "line_number": 280 + "line_number": 291 }, { "type": "Secret Keyword", "filename": "apis/clusters/v1beta1/cadence_types.go", "hashed_secret": "a57ce131bd944bdf8ba2f2f93e179dc416ed0315", "is_verified": false, - "line_number": 290 + "line_number": 300 } ], "apis/clusters/v1beta1/cassandra_types.go": [ @@ -199,7 +199,7 @@ "filename": "apis/clusters/v1beta1/cassandra_types.go", "hashed_secret": "e0a46b27231f798fe22dc4d5d82b5feeb5dcf085", "is_verified": false, - "line_number": 412 + "line_number": 411 } ], "apis/clusters/v1beta1/cassandra_webhook.go": [ @@ -217,14 +217,14 @@ "filename": "apis/clusters/v1beta1/kafka_types.go", "hashed_secret": "964c67cddfe8e6707157152dcf319126502199dc", "is_verified": false, - "line_number": 293 + "line_number": 271 }, { "type": "Secret Keyword", "filename": "apis/clusters/v1beta1/kafka_types.go", "hashed_secret": "589a0ad3cc6bc886a00c46a22e5065c48bd8e1b2", "is_verified": false, - "line_number": 439 + "line_number": 417 } ], "apis/clusters/v1beta1/kafkaconnect_types.go": [ @@ -342,21 +342,21 @@ "filename": "apis/clusters/v1beta1/redis_types.go", "hashed_secret": "bc1c5ae5fd4a238d86261f422e62c489de408c22", "is_verified": false, - "line_number": 168 + "line_number": 169 }, { "type": "Secret Keyword", "filename": "apis/clusters/v1beta1/redis_types.go", "hashed_secret": "d62d56668a8c859e768e8250ed2fb690d03cead3", "is_verified": false, - "line_number": 223 + "line_number": 225 }, { "type": "Secret Keyword", "filename": "apis/clusters/v1beta1/redis_types.go", "hashed_secret": "d0e8e6fc5dce4d2b452e344ae41900b566ac01d1", "is_verified": false, - "line_number": 268 + "line_number": 270 } ], "apis/clusters/v1beta1/redis_webhook.go": [ @@ -390,7 +390,7 @@ "filename": "apis/clusters/v1beta1/zz_generated.deepcopy.go", "hashed_secret": "44e17306b837162269a410204daaa5ecee4ec22c", "is_verified": false, - "line_number": 1322 + "line_number": 1316 } ], "apis/kafkamanagement/v1beta1/kafkauser_types.go": [ @@ -531,14 +531,14 @@ "filename": "controllers/clusters/cadence_controller.go", "hashed_secret": "bcf196cdeea4d7ed8b04dcbbd40111eb5e9abeac", "is_verified": false, - "line_number": 660 + "line_number": 644 }, { "type": "Secret Keyword", "filename": "controllers/clusters/cadence_controller.go", "hashed_secret": "192d703e91a60432ce06bfe26adfd12f5c7b931f", "is_verified": false, - "line_number": 701 + "line_number": 677 } ], "controllers/clusters/datatest/kafka_v1beta1.yaml": [ @@ -739,7 +739,7 @@ "filename": "pkg/instaclustr/client.go", "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8", "is_verified": false, - "line_number": 2060 + "line_number": 2072 } ], "pkg/instaclustr/mock/client.go": [ @@ -1146,5 +1146,5 @@ } ] }, - "generated_at": "2024-02-20T12:39:50Z" + "generated_at": "2024-02-21T14:33:43Z" } diff --git a/apis/clusters/v1beta1/cadence_types.go b/apis/clusters/v1beta1/cadence_types.go index 579336a79..1ef9cf7df 100644 --- a/apis/clusters/v1beta1/cadence_types.go +++ b/apis/clusters/v1beta1/cadence_types.go @@ -18,7 +18,6 @@ package v1beta1 import ( "context" - "encoding/json" "fmt" k8scorev1 "k8s.io/api/core/v1" @@ -28,12 +27,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/instaclustr/operator/pkg/models" + "github.com/instaclustr/operator/pkg/utils/slices" ) type CadenceDataCentre struct { - DataCentre `json:",inline"` - ClientEncryption bool `json:"clientEncryption,omitempty"` - PrivateLink []*PrivateLink `json:"privateLink,omitempty"` + GenericDataCentreSpec `json:",inline"` + + NumberOfNodes int `json:"numberOfNodes"` + NodeSize string `json:"nodeSize"` + ClientEncryption bool `json:"clientEncryption,omitempty"` + + PrivateLink PrivateLinkSpec `json:"privateLink,omitempty"` } type BundledCassandraSpec struct { @@ -61,18 +65,32 @@ type BundledOpenSearchSpec struct { // CadenceSpec defines the desired state of Cadence type CadenceSpec struct { - Cluster `json:",inline"` + GenericClusterSpec `json:",inline"` + //+kubebuilder:validation:MinItems:=1 //+kubebuilder:validation:MaxItems:=1 - DataCentres []*CadenceDataCentre `json:"dataCentres"` - UseCadenceWebAuth bool `json:"useCadenceWebAuth"` - AWSArchival []*AWSArchival `json:"awsArchival,omitempty"` + DataCentres []*CadenceDataCentre `json:"dataCentres"` + + //+kubebuilder:validation:MaxItems:=1 + AWSArchival []*AWSArchival `json:"awsArchival,omitempty"` + + //+kubebuilder:validation:MaxItems:=1 StandardProvisioning []*StandardProvisioning `json:"standardProvisioning,omitempty"` - SharedProvisioning []*SharedProvisioning `json:"sharedProvisioning,omitempty"` + + //+kubebuilder:validation:MaxItems:=1 + SharedProvisioning []*SharedProvisioning `json:"sharedProvisioning,omitempty"` + + //+kubebuilder:validation:MaxItems:=1 PackagedProvisioning []*PackagedProvisioning `json:"packagedProvisioning,omitempty"` - TargetPrimaryCadence []*TargetCadence `json:"targetPrimaryCadence,omitempty"` - ResizeSettings []*ResizeSettings `json:"resizeSettings,omitempty"` - UseHTTPAPI bool `json:"useHttpApi,omitempty"` + + //+kubebuilder:validation:MaxItems:=1 + TargetPrimaryCadence []*CadenceDependencyTarget `json:"targetPrimaryCadence,omitempty"` + + ResizeSettings GenericResizeSettings `json:"resizeSettings,omitempty"` + + UseCadenceWebAuth bool `json:"useCadenceWebAuth"` + UseHTTPAPI bool `json:"useHttpApi,omitempty"` + PCICompliance bool `json:"pciCompliance,omitempty"` } type AWSArchival struct { @@ -94,39 +112,36 @@ type SharedProvisioning struct { } type StandardProvisioning struct { - AdvancedVisibility []*AdvancedVisibility `json:"advancedVisibility,omitempty"` - TargetCassandra *TargetCassandra `json:"targetCassandra"` + //+kubebuilder:validation:MaxItems:=1 + AdvancedVisibility []*AdvancedVisibility `json:"advancedVisibility,omitempty"` + TargetCassandra *CadenceDependencyTarget `json:"targetCassandra"` } -type TargetCassandra struct { +type CadenceDependencyTarget struct { DependencyCDCID string `json:"dependencyCdcId"` DependencyVPCType string `json:"dependencyVpcType"` } -type TargetKafka struct { - DependencyCDCID string `json:"dependencyCdcId"` - DependencyVPCType string `json:"dependencyVpcType"` +type AdvancedVisibility struct { + TargetKafka *CadenceDependencyTarget `json:"targetKafka"` + TargetOpenSearch *CadenceDependencyTarget `json:"targetOpenSearch"` } -type TargetOpenSearch struct { - DependencyCDCID string `json:"dependencyCdcId"` - DependencyVPCType string `json:"dependencyVpcType"` -} +// CadenceStatus defines the observed state of Cadence +type CadenceStatus struct { + GenericStatus `json:",inline"` -type TargetCadence struct { - DependencyCDCID string `json:"dependencyCdcId"` - DependencyVPCType string `json:"dependencyVpcType"` + DataCentres []*CadenceDataCentreStatus `json:"dataCentres,omitempty"` + TargetSecondaryCadence []*CadenceDependencyTarget `json:"targetSecondaryCadence,omitempty"` } -type AdvancedVisibility struct { - TargetKafka *TargetKafka `json:"targetKafka"` - TargetOpenSearch *TargetOpenSearch `json:"targetOpenSearch"` -} +type CadenceDataCentreStatus struct { + GenericDataCentreStatus `json:",inline"` -// CadenceStatus defines the observed state of Cadence -type CadenceStatus struct { - ClusterStatus `json:",inline"` - TargetSecondaryCadence []*TargetCadence `json:"targetSecondaryCadence,omitempty"` + NumberOfNodes int `json:"numberOfNodes,omitempty"` + + Nodes []*Node `json:"nodes,omitempty"` + PrivateLink PrivateLinkStatuses `json:"privateLink,omitempty"` } //+kubebuilder:object:root=true @@ -197,21 +212,18 @@ func (cs *CadenceSpec) ToInstAPI(ctx context.Context, k8sClient client.Client) ( } return &models.CadenceCluster{ - CadenceVersion: cs.Version, - DataCentres: cs.DCsToInstAPI(), - Name: cs.Name, - PCIComplianceMode: cs.PCICompliance, - TwoFactorDelete: cs.TwoFactorDeletesToInstAPI(), - UseCadenceWebAuth: cs.UseCadenceWebAuth, - PrivateNetworkCluster: cs.PrivateNetworkCluster, - SLATier: cs.SLATier, - AWSArchival: awsArchival, - Description: cs.Description, - SharedProvisioning: sharedProvisioning, - StandardProvisioning: standardProvisioning, - TargetPrimaryCadence: cs.TargetCadenceToInstAPI(), - ResizeSettings: resizeSettingsToInstAPI(cs.ResizeSettings), - UseHTTPAPI: cs.UseHTTPAPI, + GenericClusterFields: cs.GenericClusterSpec.ToInstAPI(), + CadenceVersion: cs.Version, + DataCentres: cs.DCsToInstAPI(), + PCIComplianceMode: cs.PCICompliance, + TwoFactorDelete: cs.TwoFactorDeleteToInstAPI(), + UseCadenceWebAuth: cs.UseCadenceWebAuth, + AWSArchival: awsArchival, + SharedProvisioning: sharedProvisioning, + StandardProvisioning: standardProvisioning, + TargetPrimaryCadence: cs.TargetCadenceToInstAPI(), + ResizeSettings: resizeSettingsToInstAPI(cs.ResizeSettings), + UseHTTPAPI: cs.UseHTTPAPI, }, nil } @@ -222,7 +234,7 @@ func (cs *CadenceSpec) StandardProvisioningToInstAPI() []*models.CadenceStandard targetCassandra := standardProvisioning.TargetCassandra stdProvisioning := &models.CadenceStandardProvisioning{ - TargetCassandra: &models.TargetCassandra{ + TargetCassandra: &models.CadenceDependencyTarget{ DependencyCDCID: targetCassandra.DependencyCDCID, DependencyVPCType: targetCassandra.DependencyVPCType, }, @@ -234,11 +246,11 @@ func (cs *CadenceSpec) StandardProvisioningToInstAPI() []*models.CadenceStandard stdProvisioning.AdvancedVisibility = []*models.AdvancedVisibility{ { - TargetKafka: &models.TargetKafka{ + TargetKafka: &models.CadenceDependencyTarget{ DependencyCDCID: targetKafka.DependencyCDCID, DependencyVPCType: targetKafka.DependencyVPCType, }, - TargetOpenSearch: &models.TargetOpenSearch{ + TargetOpenSearch: &models.CadenceDependencyTarget{ DependencyCDCID: targetOpenSearch.DependencyCDCID, DependencyVPCType: targetOpenSearch.DependencyVPCType, }, @@ -285,62 +297,39 @@ func (cs *CadenceSpec) ArchivalToInstAPI(ctx context.Context, k8sClient client.C } func getSecret(ctx context.Context, k8sClient client.Client, aws *AWSArchival) (string, string, error) { - var err error awsCredsSecret := &k8scorev1.Secret{} awsSecretNamespacedName := types.NamespacedName{ Name: aws.AccessKeySecretName, Namespace: aws.AccessKeySecretNamespace, } - err = k8sClient.Get(ctx, awsSecretNamespacedName, awsCredsSecret) + + err := k8sClient.Get(ctx, awsSecretNamespacedName, awsCredsSecret) if err != nil { return "", "", err } - var AWSAccessKeyID string - var AWSSecretAccessKey string keyID := awsCredsSecret.Data[models.AWSAccessKeyID] secretAccessKey := awsCredsSecret.Data[models.AWSSecretAccessKey] - AWSAccessKeyID = string(keyID[:len(keyID)-1]) - AWSSecretAccessKey = string(secretAccessKey[:len(secretAccessKey)-1]) + AWSAccessKeyID := string(keyID[:len(keyID)-1]) + AWSSecretAccessKey := string(secretAccessKey[:len(secretAccessKey)-1]) - return AWSAccessKeyID, AWSSecretAccessKey, err + return AWSAccessKeyID, AWSSecretAccessKey, nil } func (cdc *CadenceDataCentre) ToInstAPI() *models.CadenceDataCentre { - cloudProviderSettings := cdc.CloudProviderSettingsToInstAPI() return &models.CadenceDataCentre{ - PrivateLink: cdc.privateLinkToInstAPI(), + GenericDataCentreFields: cdc.GenericDataCentreSpec.ToInstAPI(), + PrivateLink: cdc.PrivateLink.ToInstAPI(), ClientToClusterEncryption: cdc.ClientEncryption, - DataCentre: models.DataCentre{ - CloudProvider: cdc.CloudProvider, - Name: cdc.Name, - Network: cdc.Network, - NodeSize: cdc.NodeSize, - NumberOfNodes: cdc.NodesNumber, - Region: cdc.Region, - ProviderAccountName: cdc.ProviderAccountName, - AWSSettings: cloudProviderSettings.AWSSettings, - GCPSettings: cloudProviderSettings.GCPSettings, - AzureSettings: cloudProviderSettings.AzureSettings, - Tags: cdc.TagsToInstAPI(), - }, + NumberOfNodes: cdc.NumberOfNodes, + NodeSize: cdc.NodeSize, } } -func (cd *CadenceDataCentre) privateLinkToInstAPI() (iPrivateLink []*models.PrivateLink) { - for _, link := range cd.PrivateLink { - iPrivateLink = append(iPrivateLink, &models.PrivateLink{ - AdvertisedHostname: link.AdvertisedHostname, - }) - } - - return -} - -func (cs *CadenceSpec) TargetCadenceToInstAPI() []*models.TargetCadence { - var targets []*models.TargetCadence +func (cs *CadenceSpec) TargetCadenceToInstAPI() []*models.CadenceDependencyTarget { + var targets []*models.CadenceDependencyTarget for _, target := range cs.TargetPrimaryCadence { - targets = append(targets, &models.TargetCadence{ + targets = append(targets, &models.CadenceDependencyTarget{ DependencyCDCID: target.DependencyCDCID, DependencyVPCType: target.DependencyVPCType, }) @@ -357,40 +346,26 @@ func (cs *CadenceSpec) DCsToInstAPI() (iDCs []*models.CadenceDataCentre) { return } -func (c *Cadence) FromInstAPI(iData []byte) (*Cadence, error) { - iCad := &models.CadenceCluster{} - err := json.Unmarshal(iData, iCad) - if err != nil { - return nil, err - } - - return &Cadence{ - TypeMeta: c.TypeMeta, - ObjectMeta: c.ObjectMeta, - Spec: c.Spec.FromInstAPI(iCad), - Status: c.Status.FromInstAPI(iCad), - }, nil +func (c *Cadence) FromInstAPI(instaModel *models.CadenceCluster) { + c.Spec.FromInstAPI(instaModel) + c.Status.FromInstAPI(instaModel) } -func (cs *CadenceSpec) FromInstAPI(iCad *models.CadenceCluster) (spec CadenceSpec) { - spec.DataCentres = cs.DCsFromInstAPI(iCad.DataCentres) - spec.ResizeSettings = resizeSettingsFromInstAPI(iCad.ResizeSettings) - spec.TwoFactorDelete = cs.Cluster.TwoFactorDeleteFromInstAPI(iCad.TwoFactorDelete) - spec.Description = iCad.Description +func (cs *CadenceSpec) FromInstAPI(instaModel *models.CadenceCluster) { + cs.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields, instaModel.CadenceVersion) + cs.ResizeSettings.FromInstAPI(instaModel.ResizeSettings) - return + cs.DCsFromInstAPI(instaModel.DataCentres) } -func (cs *CadenceSpec) DCsFromInstAPI(iDCs []*models.CadenceDataCentre) (dcs []*CadenceDataCentre) { - for _, iDC := range iDCs { - dcs = append(dcs, &CadenceDataCentre{ - DataCentre: cs.Cluster.DCFromInstAPI(iDC.DataCentre), - ClientEncryption: iDC.ClientToClusterEncryption, - PrivateLink: cs.PrivateLinkFromInstAPI(iDC.PrivateLink), - }) +func (cs *CadenceSpec) DCsFromInstAPI(instaModels []*models.CadenceDataCentre) { + dcs := make([]*CadenceDataCentre, len(instaModels)) + for i, instaModel := range instaModels { + dc := &CadenceDataCentre{} + dc.FromInstAPI(instaModel) + dcs[i] = dc } - - return + cs.DataCentres = dcs } func (cs *CadenceSpec) PrivateLinkFromInstAPI(iPLs []*models.PrivateLink) (pls []*PrivateLink) { @@ -405,28 +380,27 @@ func (cs *CadenceSpec) PrivateLinkFromInstAPI(iPLs []*models.PrivateLink) (pls [ func (c *Cadence) GetSpec() CadenceSpec { return c.Spec } func (c *Cadence) IsSpecEqual(spec CadenceSpec) bool { - return c.Spec.IsEqual(spec) + return c.Spec.Equals(&spec.GenericClusterSpec) && + c.Spec.DCsEqual(spec.DataCentres) } -func (cs *CadenceSpec) IsEqual(spec CadenceSpec) bool { - return cs.AreDCsEqual(spec.DataCentres) -} - -func (cs *CadenceSpec) AreDCsEqual(dcs []*CadenceDataCentre) bool { +func (cs *CadenceSpec) DCsEqual(dcs []*CadenceDataCentre) bool { if len(cs.DataCentres) != len(dcs) { return false } - for i, iDC := range dcs { - dataCentre := cs.DataCentres[i] + m := map[string]*CadenceDataCentre{} + for _, dc := range cs.DataCentres { + m[dc.Name] = dc + } - if iDC.Name != dataCentre.Name { - continue + for _, dc := range dcs { + mDC, ok := m[dc.Name] + if !ok { + return false } - if !dataCentre.IsEqual(iDC.DataCentre) || - iDC.ClientEncryption != dataCentre.ClientEncryption || - !arePrivateLinksEqual(dataCentre.PrivateLink, iDC.PrivateLink) { + if !mDC.Equals(dc) { return false } } @@ -434,50 +408,41 @@ func (cs *CadenceSpec) AreDCsEqual(dcs []*CadenceDataCentre) bool { return true } -func (cs *CadenceStatus) FromInstAPI(iCad *models.CadenceCluster) CadenceStatus { - return CadenceStatus{ - ClusterStatus: ClusterStatus{ - ID: iCad.ID, - State: iCad.Status, - DataCentres: cs.DCsFromInstAPI(iCad.DataCentres), - CurrentClusterOperationStatus: iCad.CurrentClusterOperationStatus, - MaintenanceEvents: cs.MaintenanceEvents, - }, - TargetSecondaryCadence: cs.SecondaryTargetsFromInstAPI(iCad), - } +func (cs *CadenceStatus) FromInstAPI(instaModel *models.CadenceCluster) { + cs.GenericStatus.FromInstAPI(&instaModel.GenericClusterFields) + cs.targetSecondaryFromInstAPI(instaModel.TargetSecondaryCadence) + cs.DCsFromInstAPI(instaModel.DataCentres) } -func (cs *CadenceStatus) SecondaryTargetsFromInstAPI(iCad *models.CadenceCluster) []*TargetCadence { - var targets []*TargetCadence - for _, target := range iCad.TargetSecondaryCadence { - targets = append(targets, &TargetCadence{ +func (cs *CadenceStatus) targetSecondaryFromInstAPI(instaModels []*models.CadenceDependencyTarget) { + cs.TargetSecondaryCadence = make([]*CadenceDependencyTarget, len(instaModels)) + for i, target := range instaModels { + cs.TargetSecondaryCadence[i] = &CadenceDependencyTarget{ DependencyCDCID: target.DependencyCDCID, DependencyVPCType: target.DependencyVPCType, - }) + } } - - return targets } -func (cs *CadenceStatus) DCsFromInstAPI(iDCs []*models.CadenceDataCentre) (dcs []*DataCentreStatus) { - for _, iDC := range iDCs { - dc := cs.ClusterStatus.DCFromInstAPI(iDC.DataCentre) - dc.PrivateLink = privateLinkStatusesFromInstAPI(iDC.PrivateLink) - dcs = append(dcs, dc) +func (cs *CadenceStatus) DCsFromInstAPI(instaModels []*models.CadenceDataCentre) { + cs.DataCentres = make([]*CadenceDataCentreStatus, len(instaModels)) + for i, instaModel := range instaModels { + dc := &CadenceDataCentreStatus{} + dc.FromInstAPI(instaModel) + cs.DataCentres[i] = dc } - - return } func (cs *CadenceSpec) NewDCsUpdate() models.CadenceClusterAPIUpdate { return models.CadenceClusterAPIUpdate{ - DataCentres: cs.DCsToInstAPI(), + DataCentres: cs.DCsToInstAPI(), + ResizeSettings: cs.ResizeSettings.ToInstAPI(), } } func (c *Cadence) GetExposePorts() []k8scorev1.ServicePort { var exposePorts []k8scorev1.ServicePort - if !c.Spec.PrivateNetworkCluster { + if !c.Spec.PrivateNetwork { exposePorts = []k8scorev1.ServicePort{ { Name: models.CadenceTChannel, @@ -543,3 +508,64 @@ func (c *Cadence) GetHeadlessPorts() []k8scorev1.ServicePort { } return headlessPorts } + +func (cdc *CadenceDataCentreStatus) FromInstAPI(instaModel *models.CadenceDataCentre) { + cdc.GenericDataCentreStatus.FromInstAPI(&instaModel.GenericDataCentreFields) + cdc.NumberOfNodes = instaModel.NumberOfNodes + cdc.Nodes = nodesFromInstAPI(instaModel.Nodes) + cdc.PrivateLink.FromInstAPI(instaModel.PrivateLink) +} + +func (cdc *CadenceDataCentre) Equals(o *CadenceDataCentre) bool { + return cdc.GenericDataCentreSpec.Equals(&o.GenericDataCentreSpec) && + cdc.NumberOfNodes == o.NumberOfNodes && + cdc.NodeSize == o.NodeSize && + cdc.ClientEncryption == o.ClientEncryption && + slices.EqualsPtr(cdc.PrivateLink, o.PrivateLink) +} + +func (cdc *CadenceDataCentre) FromInstAPI(instaModel *models.CadenceDataCentre) { + cdc.GenericDataCentreSpec.FromInstAPI(&instaModel.GenericDataCentreFields) + cdc.PrivateLink.FromInstAPI(instaModel.PrivateLink) + + cdc.NumberOfNodes = instaModel.NumberOfNodes + cdc.NodeSize = instaModel.NodeSize + cdc.ClientEncryption = instaModel.ClientToClusterEncryption +} + +func (cs *CadenceStatus) Equals(o *CadenceStatus) bool { + return cs.GenericStatus.Equals(&o.GenericStatus) && + cs.DCsEqual(o.DataCentres) && + slices.EqualsPtr(cs.TargetSecondaryCadence, o.TargetSecondaryCadence) +} + +func (cs *CadenceStatus) DCsEqual(o []*CadenceDataCentreStatus) bool { + if len(cs.DataCentres) != len(o) { + return false + } + + m := map[string]*CadenceDataCentreStatus{} + for _, dc := range cs.DataCentres { + m[dc.Name] = dc + } + + for _, dc := range o { + mDC, ok := m[dc.Name] + if !ok { + return false + } + + if !mDC.Equals(dc) { + return false + } + } + + return true +} + +func (cdc *CadenceDataCentreStatus) Equals(o *CadenceDataCentreStatus) bool { + return cdc.GenericDataCentreStatus.Equals(&o.GenericDataCentreStatus) && + cdc.PrivateLink.Equal(o.PrivateLink) && + nodesEqual(cdc.Nodes, o.Nodes) && + cdc.NumberOfNodes == o.NumberOfNodes +} diff --git a/apis/clusters/v1beta1/cadence_webhook.go b/apis/clusters/v1beta1/cadence_webhook.go index 3c7398fb1..cda1a6dce 100644 --- a/apis/clusters/v1beta1/cadence_webhook.go +++ b/apis/clusters/v1beta1/cadence_webhook.go @@ -19,6 +19,7 @@ package v1beta1 import ( "context" "fmt" + "net" "regexp" "k8s.io/apimachinery/pkg/runtime" @@ -67,7 +68,7 @@ func (c *Cadence) Default() { } for _, dataCentre := range c.Spec.DataCentres { - dataCentre.SetDefaultValues() + dataCentre.ProviderAccountName = models.DefaultAccountName } } @@ -90,7 +91,7 @@ func (cv *cadenceValidator) ValidateCreate(ctx context.Context, obj runtime.Obje return err } - err = c.Spec.Cluster.ValidateCreation() + err = c.Spec.GenericClusterSpec.ValidateCreation() if err != nil { return err } @@ -114,22 +115,6 @@ func (cv *cadenceValidator) ValidateCreate(ctx context.Context, obj runtime.Obje return fmt.Errorf("only one of StandardProvisioning, SharedProvisioning or PackagedProvisioning arrays must not be empty") } - if len(c.Spec.AWSArchival) > 1 { - return fmt.Errorf("AWSArchival array size must be between 0 and 1") - } - - if len(c.Spec.StandardProvisioning) > 1 { - return fmt.Errorf("StandardProvisioning array size must be between 0 and 1") - } - - if len(c.Spec.SharedProvisioning) > 1 { - return fmt.Errorf("SharedProvisioning array size must be between 0 and 1") - } - - if len(c.Spec.PackagedProvisioning) > 1 { - return fmt.Errorf("PackagedProvisioning array size must be between 0 and 1") - } - for _, awsArchival := range c.Spec.AWSArchival { err = awsArchival.validate() if err != nil { @@ -138,58 +123,24 @@ func (cv *cadenceValidator) ValidateCreate(ctx context.Context, obj runtime.Obje } for _, sp := range c.Spec.StandardProvisioning { - if len(sp.AdvancedVisibility) > 1 { - return fmt.Errorf("AdvancedVisibility array size must be between 0 and 1") - } - err = sp.validate() if err != nil { return err } } - for _, pp := range c.Spec.PackagedProvisioning { - if (pp.UseAdvancedVisibility && pp.BundledKafkaSpec == nil) || (pp.UseAdvancedVisibility && pp.BundledOpenSearchSpec == nil) { - return fmt.Errorf("BundledKafkaSpec and BundledOpenSearchSpec structs must not be empty because UseAdvancedVisibility is set to true") - } - - if pp.BundledKafkaSpec != nil { - err = pp.BundledKafkaSpec.validate() - if err != nil { - return err - } - } - - if pp.BundledCassandraSpec != nil { - err = pp.BundledCassandraSpec.validate() - if err != nil { - return err - } - } - - if pp.BundledOpenSearchSpec != nil { - err = pp.BundledOpenSearchSpec.validate() - if err != nil { - return err - } - } - } - - if len(c.Spec.TargetPrimaryCadence) > 1 { - return fmt.Errorf("targetPrimaryCadence array must consist of <= 1 elements") - } - - if len(c.Spec.DataCentres) == 0 { - return fmt.Errorf("data centres field is empty") + err = c.Spec.validatePackagedProvisioningCreation() + if err != nil { + return err } for _, dc := range c.Spec.DataCentres { - err = dc.DataCentre.ValidateCreation() + err = dc.GenericDataCentreSpec.validateCreation() if err != nil { return err } - if !c.Spec.PrivateNetworkCluster && dc.PrivateLink != nil { + if !c.Spec.PrivateNetwork && dc.PrivateLink != nil { return models.ErrPrivateLinkOnlyWithPrivateNetworkCluster } @@ -276,7 +227,7 @@ type immutableCadenceDCFields struct { func (cs *CadenceSpec) newImmutableFields() *immutableCadenceFields { return &immutableCadenceFields{ - immutableCluster: cs.Cluster.newImmutableFields(), + immutableCluster: cs.GenericClusterSpec.immutableFields(), UseCadenceWebAuth: cs.UseCadenceWebAuth, UseHTTPAPI: cs.UseHTTPAPI, } @@ -465,11 +416,11 @@ func (cs *CadenceSpec) validateImmutableDataCentresFieldsUpdate(oldSpec CadenceS return fmt.Errorf("cannot update immutable data centre fields: new spec: %v: old spec: %v", newDCImmutableFields, oldDCImmutableFields) } - if newDC.NodesNumber < oldDC.NodesNumber { - return fmt.Errorf("deleting nodes is not supported. Number of nodes must be greater than: %v", oldDC.NodesNumber) + if newDC.NumberOfNodes < oldDC.NumberOfNodes { + return fmt.Errorf("deleting nodes is not supported. Number of nodes must be greater than: %v", oldDC.NumberOfNodes) } - err := newDC.validateImmutableCloudProviderSettingsUpdate(oldDC.CloudProviderSettings) + err := newDC.validateImmutableCloudProviderSettingsUpdate(&oldDC.GenericDataCentreSpec) if err != nil { return err } @@ -605,3 +556,69 @@ func (o *BundledOpenSearchSpec) validate() error { return nil } + +func (cs *CadenceSpec) validatePackagedProvisioningCreation() error { + for _, dc := range cs.DataCentres { + for _, pp := range cs.PackagedProvisioning { + if (pp.UseAdvancedVisibility && pp.BundledKafkaSpec == nil) || (pp.UseAdvancedVisibility && pp.BundledOpenSearchSpec == nil) { + return fmt.Errorf("BundledKafkaSpec and BundledOpenSearchSpec structs must not be empty because UseAdvancedVisibility is set to true") + } + + if pp.BundledKafkaSpec != nil { + err := pp.BundledKafkaSpec.validate() + if err != nil { + return err + } + + err = dc.validateNetwork(pp.BundledKafkaSpec.Network) + if err != nil { + return err + } + } + + if pp.BundledCassandraSpec != nil { + err := pp.BundledCassandraSpec.validate() + if err != nil { + return err + } + + err = dc.validateNetwork(pp.BundledCassandraSpec.Network) + if err != nil { + return err + } + } + + if pp.BundledOpenSearchSpec != nil { + err := pp.BundledOpenSearchSpec.validate() + if err != nil { + return err + } + + err = dc.validateNetwork(pp.BundledOpenSearchSpec.Network) + if err != nil { + return err + } + } + } + } + + return nil +} + +func (cdc *CadenceDataCentre) validateNetwork(network string) error { + _, ipnet, err := net.ParseCIDR(cdc.Network) + if err != nil { + return err + } + + ip, _, err := net.ParseCIDR(network) + if err != nil { + return err + } + + if ipnet.Contains(ip) { + return fmt.Errorf("cluster network %s overlaps with network %s", cdc.Network, network) + } + + return nil +} diff --git a/apis/clusters/v1beta1/cassandra_types.go b/apis/clusters/v1beta1/cassandra_types.go index afd326d34..ed52531f6 100644 --- a/apis/clusters/v1beta1/cassandra_types.go +++ b/apis/clusters/v1beta1/cassandra_types.go @@ -318,12 +318,11 @@ func (cs *CassandraSpec) DCsUpdateToInstAPI() models.CassandraClusterAPIUpdate { } func (cs *CassandraSpec) FromInstAPI(instModel *models.CassandraCluster) { - cs.GenericClusterSpec.FromInstAPI(&instModel.GenericClusterFields) + cs.GenericClusterSpec.FromInstAPI(&instModel.GenericClusterFields, instModel.CassandraVersion) cs.LuceneEnabled = instModel.LuceneEnabled cs.PasswordAndUserAuth = instModel.PasswordAndUserAuth cs.BundledUseOnly = instModel.BundledUseOnly - cs.Version = instModel.CassandraVersion cs.PCICompliance = instModel.PCIComplianceMode cs.ResizeSettings.FromInstAPI(instModel.ResizeSettings) diff --git a/apis/clusters/v1beta1/generic_spec.go b/apis/clusters/v1beta1/generic_spec.go index d2ea520c5..d35cbc431 100644 --- a/apis/clusters/v1beta1/generic_spec.go +++ b/apis/clusters/v1beta1/generic_spec.go @@ -32,11 +32,12 @@ func (s *GenericClusterSpec) Equals(o *GenericClusterSpec) bool { slices.EqualsPtr(s.TwoFactorDelete, o.TwoFactorDelete) } -func (s *GenericClusterSpec) FromInstAPI(model *models.GenericClusterFields) { +func (s *GenericClusterSpec) FromInstAPI(model *models.GenericClusterFields, version string) { s.Name = model.Name s.PrivateNetwork = model.PrivateNetworkCluster s.SLATier = model.SLATier s.Description = model.Description + s.Version = version s.TwoFactorDeleteFromInstAPI(model.TwoFactorDelete) } @@ -204,7 +205,7 @@ func (s *GenericDataCentreSpec) cloudProviderSettingsFromInstAPI(instaModel *mod } switch { - case len(instaModel.AWSSettings) > 0: + case len(instaModel.AWSSettings) > 0 && instaModel.HasAWSCloudProviderSettings(): setting := instaModel.AWSSettings[0] s.AWSSettings = []*AWSSettings{{ DiskEncryptionKey: setting.EBSEncryptionKey, diff --git a/apis/clusters/v1beta1/kafka_types.go b/apis/clusters/v1beta1/kafka_types.go index e1f6ee8ab..c9a134e86 100644 --- a/apis/clusters/v1beta1/kafka_types.go +++ b/apis/clusters/v1beta1/kafka_types.go @@ -353,7 +353,7 @@ func (k *Kafka) FromInstAPI(instaModel *models.KafkaCluster) { } func (ks *KafkaSpec) FromInstAPI(instaModel *models.KafkaCluster) { - ks.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields) + ks.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields, instaModel.KafkaVersion) ks.ResizeSettings.FromInstAPI(instaModel.ResizeSettings) ks.ReplicationFactor = instaModel.DefaultReplicationFactor @@ -363,7 +363,6 @@ func (ks *KafkaSpec) FromInstAPI(instaModel *models.KafkaCluster) { ks.ClientToClusterEncryption = instaModel.ClientToClusterEncryption ks.ClientBrokerAuthWithMTLS = instaModel.ClientBrokerAuthWithMtls ks.BundledUseOnly = instaModel.BundledUseOnly - ks.Version = instaModel.KafkaVersion ks.PCICompliance = instaModel.PCIComplianceMode ks.DCsFromInstAPI(instaModel.DataCentres) diff --git a/apis/clusters/v1beta1/kafkaconnect_types.go b/apis/clusters/v1beta1/kafkaconnect_types.go index 8ae043e6a..3a7b2b0a7 100644 --- a/apis/clusters/v1beta1/kafkaconnect_types.go +++ b/apis/clusters/v1beta1/kafkaconnect_types.go @@ -197,7 +197,7 @@ func (k *KafkaConnect) FromInstAPI(instaModel *models.KafkaConnectCluster) { } func (ks *KafkaConnectSpec) FromInstAPI(instaModel *models.KafkaConnectCluster) { - ks.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields) + ks.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields, instaModel.KafkaConnectVersion) ks.Version = instaModel.KafkaConnectVersion diff --git a/apis/clusters/v1beta1/opensearch_types.go b/apis/clusters/v1beta1/opensearch_types.go index 11b4cf76b..4c482b047 100644 --- a/apis/clusters/v1beta1/opensearch_types.go +++ b/apis/clusters/v1beta1/opensearch_types.go @@ -58,9 +58,8 @@ type OpenSearchSpec struct { } func (s *OpenSearchSpec) FromInstAPI(instaModel *models.OpenSearchCluster) { - s.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields) + s.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields, instaModel.OpenSearchVersion) - s.Version = instaModel.OpenSearchVersion s.ICUPlugin = instaModel.ICUPlugin s.AsynchronousSearchPlugin = instaModel.AsynchronousSearchPlugin s.KNNPlugin = instaModel.KNNPlugin diff --git a/apis/clusters/v1beta1/redis_types.go b/apis/clusters/v1beta1/redis_types.go index 33f52d276..15d2dd0ee 100644 --- a/apis/clusters/v1beta1/redis_types.go +++ b/apis/clusters/v1beta1/redis_types.go @@ -264,11 +264,10 @@ func (r *Redis) FromInstAPI(instaModel *models.RedisCluster) { } func (rs *RedisSpec) FromInstAPI(instaModel *models.RedisCluster) { - rs.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields) + rs.GenericClusterSpec.FromInstAPI(&instaModel.GenericClusterFields, instaModel.RedisVersion) rs.ClientEncryption = instaModel.ClientToNodeEncryption rs.PasswordAndUserAuth = instaModel.PasswordAndUserAuth - rs.Version = instaModel.RedisVersion rs.PCICompliance = instaModel.PCIComplianceMode rs.DCsFromInstAPI(instaModel.DataCentres) diff --git a/apis/clusters/v1beta1/structs.go b/apis/clusters/v1beta1/structs.go index 09e8fe002..c7a3f55d0 100644 --- a/apis/clusters/v1beta1/structs.go +++ b/apis/clusters/v1beta1/structs.go @@ -18,7 +18,6 @@ package v1beta1 import ( "encoding/json" - "net" clusterresource "github.com/instaclustr/operator/apis/clusterresources/v1beta1" "github.com/instaclustr/operator/pkg/apiextensions" @@ -147,7 +146,7 @@ type PrivateLink struct { type PrivateLinkSpec []*PrivateLink func (p PrivateLinkSpec) ToInstAPI() []*models.PrivateLink { - instaModels := make([]*models.PrivateLink, len(p)) + instaModels := make([]*models.PrivateLink, 0, len(p)) for _, pl := range p { instaModels = append(instaModels, &models.PrivateLink{ AdvertisedHostname: pl.AdvertisedHostname, @@ -223,19 +222,6 @@ func privateLinksToInstAPI(p []*PrivateLink) []*models.PrivateLink { return links } -func privateLinkStatusesFromInstAPI(iPLs []*models.PrivateLink) PrivateLinkStatuses { - k8sPLs := make(PrivateLinkStatuses, 0, len(iPLs)) - for _, link := range iPLs { - k8sPLs = append(k8sPLs, &privateLinkStatus{ - AdvertisedHostname: link.AdvertisedHostname, - EndPointServiceID: link.EndPointServiceID, - EndPointServiceName: link.EndPointServiceName, - }) - } - - return k8sPLs -} - type PrivateLinkV1 struct { IAMPrincipalARNs []string `json:"iamPrincipalARNs"` } @@ -295,19 +281,6 @@ func resizeSettingsToInstAPI(rss []*ResizeSettings) []*models.ResizeSettings { return iRS } -func resizeSettingsFromInstAPI(rss []*models.ResizeSettings) []*ResizeSettings { - iRS := make([]*ResizeSettings, 0, len(rss)) - - for _, rs := range rss { - iRS = append(iRS, &ResizeSettings{ - NotifySupportContacts: rs.NotifySupportContacts, - Concurrency: rs.Concurrency, - }) - } - - return iRS -} - type ReplaceOperation struct { // ID of the new node in the replacement operation NewNodeID string `json:"newNodeId,omitempty"` @@ -384,24 +357,6 @@ func (c *Cluster) TwoFactorDeleteToInstAPIv1() *models.TwoFactorDeleteV1 { } } -func (dc *DataCentre) IsNetworkOverlaps(networkToCheck string) (bool, error) { - _, ipnet, err := net.ParseCIDR(dc.Network) - if err != nil { - return false, err - } - - cassIP, _, err := net.ParseCIDR(networkToCheck) - if err != nil { - return false, err - } - - if ipnet.Contains(cassIP) { - return true, nil - } - - return false, nil -} - func (dc *DataCentre) ToInstAPI() models.DataCentre { providerSettings := dc.CloudProviderSettingsToInstAPI() return models.DataCentre{ @@ -730,20 +685,6 @@ func (cs *ClusterStatus) NodesFromInstAPIv1(iNodes []*models.NodeStatusV1) (node return nodes } -func arePrivateLinksEqual(a, b []*PrivateLink) bool { - if len(a) != len(b) { - return false - } - - for i, privateLink := range a { - if *b[i] != *privateLink { - return false - } - } - - return true -} - func (cs *ClusterStatus) PrivateLinkStatusesEqual(iStatus *ClusterStatus) bool { for _, iDC := range iStatus.DataCentres { for _, k8sDC := range cs.DataCentres { diff --git a/apis/clusters/v1beta1/zz_generated.deepcopy.go b/apis/clusters/v1beta1/zz_generated.deepcopy.go index 204f11f2d..d46e85410 100644 --- a/apis/clusters/v1beta1/zz_generated.deepcopy.go +++ b/apis/clusters/v1beta1/zz_generated.deepcopy.go @@ -78,12 +78,12 @@ func (in *AdvancedVisibility) DeepCopyInto(out *AdvancedVisibility) { *out = *in if in.TargetKafka != nil { in, out := &in.TargetKafka, &out.TargetKafka - *out = new(TargetKafka) + *out = new(CadenceDependencyTarget) **out = **in } if in.TargetOpenSearch != nil { in, out := &in.TargetOpenSearch, &out.TargetOpenSearch - *out = new(TargetOpenSearch) + *out = new(CadenceDependencyTarget) **out = **in } } @@ -203,10 +203,10 @@ func (in *Cadence) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CadenceDataCentre) DeepCopyInto(out *CadenceDataCentre) { *out = *in - in.DataCentre.DeepCopyInto(&out.DataCentre) + in.GenericDataCentreSpec.DeepCopyInto(&out.GenericDataCentreSpec) if in.PrivateLink != nil { in, out := &in.PrivateLink, &out.PrivateLink - *out = make([]*PrivateLink, len(*in)) + *out = make(PrivateLinkSpec, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] @@ -227,6 +227,59 @@ func (in *CadenceDataCentre) DeepCopy() *CadenceDataCentre { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CadenceDataCentreStatus) DeepCopyInto(out *CadenceDataCentreStatus) { + *out = *in + in.GenericDataCentreStatus.DeepCopyInto(&out.GenericDataCentreStatus) + if in.Nodes != nil { + in, out := &in.Nodes, &out.Nodes + *out = make([]*Node, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Node) + (*in).DeepCopyInto(*out) + } + } + } + if in.PrivateLink != nil { + in, out := &in.PrivateLink, &out.PrivateLink + *out = make(PrivateLinkStatuses, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(privateLinkStatus) + **out = **in + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CadenceDataCentreStatus. +func (in *CadenceDataCentreStatus) DeepCopy() *CadenceDataCentreStatus { + if in == nil { + return nil + } + out := new(CadenceDataCentreStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CadenceDependencyTarget) DeepCopyInto(out *CadenceDependencyTarget) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CadenceDependencyTarget. +func (in *CadenceDependencyTarget) DeepCopy() *CadenceDependencyTarget { + if in == nil { + return nil + } + out := new(CadenceDependencyTarget) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CadenceList) DeepCopyInto(out *CadenceList) { *out = *in @@ -262,7 +315,7 @@ func (in *CadenceList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CadenceSpec) DeepCopyInto(out *CadenceSpec) { *out = *in - in.Cluster.DeepCopyInto(&out.Cluster) + in.GenericClusterSpec.DeepCopyInto(&out.GenericClusterSpec) if in.DataCentres != nil { in, out := &in.DataCentres, &out.DataCentres *out = make([]*CadenceDataCentre, len(*in)) @@ -320,18 +373,18 @@ func (in *CadenceSpec) DeepCopyInto(out *CadenceSpec) { } if in.TargetPrimaryCadence != nil { in, out := &in.TargetPrimaryCadence, &out.TargetPrimaryCadence - *out = make([]*TargetCadence, len(*in)) + *out = make([]*CadenceDependencyTarget, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] - *out = new(TargetCadence) + *out = new(CadenceDependencyTarget) **out = **in } } } if in.ResizeSettings != nil { in, out := &in.ResizeSettings, &out.ResizeSettings - *out = make([]*ResizeSettings, len(*in)) + *out = make(GenericResizeSettings, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] @@ -355,14 +408,25 @@ func (in *CadenceSpec) DeepCopy() *CadenceSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CadenceStatus) DeepCopyInto(out *CadenceStatus) { *out = *in - in.ClusterStatus.DeepCopyInto(&out.ClusterStatus) + in.GenericStatus.DeepCopyInto(&out.GenericStatus) + if in.DataCentres != nil { + in, out := &in.DataCentres, &out.DataCentres + *out = make([]*CadenceDataCentreStatus, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(CadenceDataCentreStatus) + (*in).DeepCopyInto(*out) + } + } + } if in.TargetSecondaryCadence != nil { in, out := &in.TargetSecondaryCadence, &out.TargetSecondaryCadence - *out = make([]*TargetCadence, len(*in)) + *out = make([]*CadenceDependencyTarget, len(*in)) for i := range *in { if (*in)[i] != nil { in, out := &(*in)[i], &(*out)[i] - *out = new(TargetCadence) + *out = new(CadenceDependencyTarget) **out = **in } } @@ -2905,7 +2969,7 @@ func (in *StandardProvisioning) DeepCopyInto(out *StandardProvisioning) { } if in.TargetCassandra != nil { in, out := &in.TargetCassandra, &out.TargetCassandra - *out = new(TargetCassandra) + *out = new(CadenceDependencyTarget) **out = **in } } @@ -2920,36 +2984,6 @@ func (in *StandardProvisioning) DeepCopy() *StandardProvisioning { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TargetCadence) DeepCopyInto(out *TargetCadence) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetCadence. -func (in *TargetCadence) DeepCopy() *TargetCadence { - if in == nil { - return nil - } - out := new(TargetCadence) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TargetCassandra) DeepCopyInto(out *TargetCassandra) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetCassandra. -func (in *TargetCassandra) DeepCopy() *TargetCassandra { - if in == nil { - return nil - } - out := new(TargetCassandra) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TargetCluster) DeepCopyInto(out *TargetCluster) { *out = *in @@ -2987,36 +3021,6 @@ func (in *TargetCluster) DeepCopy() *TargetCluster { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TargetKafka) DeepCopyInto(out *TargetKafka) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetKafka. -func (in *TargetKafka) DeepCopy() *TargetKafka { - if in == nil { - return nil - } - out := new(TargetKafka) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TargetOpenSearch) DeepCopyInto(out *TargetOpenSearch) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetOpenSearch. -func (in *TargetOpenSearch) DeepCopy() *TargetOpenSearch { - if in == nil { - return nil - } - out := new(TargetOpenSearch) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TwoFactorDelete) DeepCopyInto(out *TwoFactorDelete) { *out = *in diff --git a/config/crd/bases/clusters.instaclustr.com_cadences.yaml b/config/crd/bases/clusters.instaclustr.com_cadences.yaml index 7fe360a79..a329f66ad 100644 --- a/config/crd/bases/clusters.instaclustr.com_cadences.yaml +++ b/config/crd/bases/clusters.instaclustr.com_cadences.yaml @@ -65,38 +65,118 @@ spec: - awsAccessKeySecretName - awsAccessKeySecretNamespace type: object + maxItems: 1 type: array dataCentres: items: properties: accountName: + default: INSTACLUSTR + description: For customers running in their own account. Your + provider account can be found on the Create Cluster page on + the Instaclustr Console, or the "Provider Account" property + on any existing cluster. For customers provisioning on Instaclustr's + cloud provider accounts, this property may be omitted. type: string + awsSettings: + description: AWS specific settings for the Data Centre. Cannot + be provided with GCP or Azure settings. + items: + properties: + backupBucket: + description: Specify the S3 bucket to use for storing + backup data for the cluster data centre. Only available + for customers running in their own cloud provider accounts. + Currently supported for OpenSearch clusters only. + type: string + customVirtualNetworkId: + description: VPC ID into which the Data Centre will be + provisioned. The Data Centre's network allocation must + match the IPv4 CIDR block of the specified VPC. + type: string + encryptionKey: + description: ID of a KMS encryption key to encrypt data + on nodes. KMS encryption key must be set in Cluster + Resources through the Instaclustr Console before provisioning + an encrypted Data Centre. + type: string + type: object + maxItems: 1 + type: array + azureSettings: + description: Azure specific settings for the Data Centre. Cannot + be provided with AWS or GCP settings. + items: + properties: + customVirtualNetworkId: + description: VNet ID into which the Data Centre will be + provisioned. The VNet must have an available address + space for the Data Centre's network allocation to be + appended to the VNet. Currently supported for PostgreSQL + clusters only. + type: string + resourceGroup: + description: The name of the Azure Resource Group into + which the Data Centre will be provisioned. + type: string + storageNetwork: + description: 'The private network address block to be + used for the storage network. This is only used for + certain node sizes, currently limited to those which + use Azure NetApp Files: for all other node sizes, this + field should not be provided. The network must have + a prefix length between /16 and /28, and must be part + of a private address range.' + type: string + type: object + maxItems: 1 + type: array clientEncryption: type: boolean cloudProvider: + description: Name of a cloud provider service. type: string - cloudProviderSettings: + gcpSettings: + description: GCP specific settings for the Data Centre. Cannot + be provided with AWS or Azure settings. items: properties: - backupBucket: - type: string customVirtualNetworkId: + description: "Network name or a relative Network or Subnetwork + URI. The Data Centre's network allocation must match + the IPv4 CIDR block of the specified subnet. \n Examples: + Network URI: projects/{riyoa-gcp-project-name}/global/networks/{network-name}. + Network name: {network-name}, equivalent to projects/{riyoa-gcp-project-name}/global/networks/{network-name}. + Same-project subnetwork URI: projects/{riyoa-gcp-project-name}/regions/{region-id}/subnetworks/{subnetwork-name}. + Shared VPC subnetwork URI: projects/{riyoa-gcp-host-project-name}/regions/{region-id}/subnetworks/{subnetwork-name}." type: string disableSnapshotAutoExpiry: + description: Specify whether the GCS backup bucket should + automatically expire data after 7 days or not. Setting + this to true will disable automatic expiry and will + allow for creation of custom snapshot repositories with + customisable retention using the Index Management Plugin. + The storage will have to be manually cleared after the + cluster is deleted. Only available for customers running + in their own cloud provider accounts. Currently supported + for OpenSearch clusters only. type: boolean - diskEncryptionKey: - type: string - resourceGroup: - type: string type: object + maxItems: 1 type: array name: + description: A logical name for the data centre within a cluster. + These names must be unique in the cluster. type: string network: + description: The private network address block for the Data + Centre specified using CIDR address notation. The network + must have a prefix length between /12 and /22 and must be + part of a private address space. type: string nodeSize: type: string - nodesNumber: + numberOfNodes: type: integer privateLink: items: @@ -107,18 +187,26 @@ spec: required: - advertisedHostname type: object + maxItems: 1 type: array region: + description: Region of the Data Centre. type: string tags: additionalProperties: type: string + description: List of tags to apply to the Data Centre. Tags + are metadata labels which allow you to identify, categorize + and filter clusters. This can be useful for grouping together + clusters into applications, environments, or any category + that you require. type: object required: - cloudProvider + - name - network - nodeSize - - nodesNumber + - numberOfNodes - region type: object maxItems: 1 @@ -192,14 +280,11 @@ spec: - bundledCassandraSpec - useAdvancedVisibility type: object + maxItems: 1 type: array - pciCompliance: - description: The PCI compliance standards relate to the security of - user data and transactional information. Can only be applied clusters - provisioned on AWS_VPC, running Cassandra, Kafka, Elasticsearch - and Redis. + pciComplianceMode: type: boolean - privateNetworkCluster: + privateNetwork: type: boolean resizeSettings: items: @@ -209,6 +294,7 @@ spec: notifySupportContacts: type: boolean type: object + maxItems: 1 type: array sharedProvisioning: items: @@ -218,6 +304,7 @@ spec: required: - useAdvancedVisibility type: object + maxItems: 1 type: array slaTier: description: 'Non-production clusters may receive lower priority support @@ -255,6 +342,7 @@ spec: - targetKafka - targetOpenSearch type: object + maxItems: 1 type: array targetCassandra: properties: @@ -269,6 +357,7 @@ spec: required: - targetCassandra type: object + maxItems: 1 type: array targetPrimaryCadence: items: @@ -281,6 +370,7 @@ spec: - dependencyCdcId - dependencyVpcType type: object + maxItems: 1 type: array twoFactorDelete: items: @@ -303,20 +393,17 @@ spec: type: string required: - dataCentres + - pciComplianceMode - useCadenceWebAuth type: object status: description: CadenceStatus defines the observed state of Cadence properties: - cdcid: - type: string currentClusterOperationStatus: type: string dataCentres: items: properties: - encryptionKeyId: - type: string id: type: string name: @@ -342,7 +429,7 @@ spec: type: string type: object type: array - nodesNumber: + numberOfNodes: type: integer privateLink: items: @@ -503,15 +590,6 @@ spec: type: array type: object type: array - options: - properties: - dataNodeSize: - type: string - masterNodeSize: - type: string - openSearchDashboardsNodeSize: - type: string - type: object state: type: string targetSecondaryCadence: @@ -526,8 +604,6 @@ spec: - dependencyVpcType type: object type: array - twoFactorDeleteEnabled: - type: boolean type: object type: object served: true diff --git a/config/crd/bases/clusters.instaclustr.com_cassandras.yaml b/config/crd/bases/clusters.instaclustr.com_cassandras.yaml index 5ef899082..e18d85ced 100644 --- a/config/crd/bases/clusters.instaclustr.com_cassandras.yaml +++ b/config/crd/bases/clusters.instaclustr.com_cassandras.yaml @@ -342,6 +342,8 @@ spec: type: array version: type: string + required: + - pciCompliance type: object status: description: CassandraStatus defines the observed state of Cassandra diff --git a/config/samples/clusters_v1beta1_cadence.yaml b/config/samples/clusters_v1beta1_cadence.yaml index f1c9d2aaf..73b417dbb 100644 --- a/config/samples/clusters_v1beta1_cadence.yaml +++ b/config/samples/clusters_v1beta1_cadence.yaml @@ -40,16 +40,15 @@ spec: # - email: "rostyslp@netapp.com" privateNetworkCluster: false dataCentres: - - region: "US_EAST_2" + - region: "US_EAST_1" network: "10.12.0.0/16" - # if you use multi-region mode please provide # In a multi-region mode setup, ensure the CIDR block for the secondary cluster does not overlap with the primary one # network: "10.16.0.0/16" cloudProvider: "AWS_VPC" name: "testdc" # nodeSize: "CAD-PRD-m5ad.large-75" nodeSize: "CAD-DEV-t3.small-5" - nodesNumber: 2 + numberOfNodes: 2 clientEncryption: false # privateLink: # - advertisedHostname: "cadence-sample-test.com" diff --git a/config/samples/clusters_v1beta1_cassandra.yaml b/config/samples/clusters_v1beta1_cassandra.yaml index e843da1bd..244eda769 100644 --- a/config/samples/clusters_v1beta1_cassandra.yaml +++ b/config/samples/clusters_v1beta1_cassandra.yaml @@ -1,7 +1,7 @@ apiVersion: clusters.instaclustr.com/v1beta1 kind: Cassandra metadata: - name: cassandra-cluster-2 + name: cassandra-cluster spec: name: "example-cassandra" #(immutable) version: "4.0.10" #(immutable) diff --git a/controllers/clusters/cadence_controller.go b/controllers/clusters/cadence_controller.go index 635dd69c2..872513845 100644 --- a/controllers/clusters/cadence_controller.go +++ b/controllers/clusters/cadence_controller.go @@ -234,7 +234,7 @@ func (r *CadenceReconciler) handleCreateCluster( } if c.Status.State != models.DeletedStatus { - err := r.startClusterStatusJob(c) + err := r.startSyncJob(c) if err != nil { l.Error(err, "Cannot start cluster status job", "c cluster ID", c.Status.ID, @@ -259,7 +259,7 @@ func (r *CadenceReconciler) handleUpdateCluster( req ctrl.Request, l logr.Logger, ) (ctrl.Result, error) { - iData, err := r.API.GetCadence(c.Status.ID) + instaModel, err := r.API.GetCadence(c.Status.ID) if err != nil { l.Error( err, "Cannot get Cadence cluster from the Instaclustr API", @@ -273,26 +273,15 @@ func (r *CadenceReconciler) handleUpdateCluster( return ctrl.Result{}, err } - iCadence, err := c.FromInstAPI(iData) - if err != nil { - l.Error( - err, "Cannot convert Cadence cluster from the Instaclustr API", - "cluster name", c.Spec.Name, - "cluster ID", c.Status.ID, - ) - - r.EventRecorder.Eventf(c, models.Warning, models.ConversionFailed, - "Cluster convertion from the Instaclustr API to k8s resource is failed. Reason: %v", err) - - return ctrl.Result{}, err - } + iCadence := &v1beta1.Cadence{} + iCadence.FromInstAPI(instaModel) if c.Annotations[models.ExternalChangesAnnotation] == models.True || r.RateLimiter.NumRequeues(req) == rlimiter.DefaultMaxTries { return handleExternalChanges[v1beta1.CadenceSpec](r.EventRecorder, r.Client, c, iCadence, l) } - if c.Spec.ClusterSettingsNeedUpdate(iCadence.Spec.Cluster) { + if c.Spec.ClusterSettingsNeedUpdate(&iCadence.Spec.GenericClusterSpec) { l.Info("Updating cluster settings", "instaclustr description", iCadence.Spec.Description, "instaclustr two factor delete", iCadence.Spec.TwoFactorDelete) @@ -519,8 +508,8 @@ func (r *CadenceReconciler) preparePackagedSolution( kafkaList := &v1beta1.KafkaList{} osList := &v1beta1.OpenSearchList{} advancedVisibility := &v1beta1.AdvancedVisibility{ - TargetKafka: &v1beta1.TargetKafka{}, - TargetOpenSearch: &v1beta1.TargetOpenSearch{}, + TargetKafka: &v1beta1.CadenceDependencyTarget{}, + TargetOpenSearch: &v1beta1.CadenceDependencyTarget{}, } var advancedVisibilities []*v1beta1.AdvancedVisibility if packagedProvisioning.UseAdvancedVisibility { @@ -607,7 +596,7 @@ func (r *CadenceReconciler) preparePackagedSolution( c.Spec.StandardProvisioning = append(c.Spec.StandardProvisioning, &v1beta1.StandardProvisioning{ AdvancedVisibility: advancedVisibilities, - TargetCassandra: &v1beta1.TargetCassandra{ + TargetCassandra: &v1beta1.CadenceDependencyTarget{ DependencyCDCID: cassandraList.Items[0].Status.DataCentres[0].ID, DependencyVPCType: models.VPCPeered, }, @@ -630,12 +619,8 @@ func (r *CadenceReconciler) newCassandraSpec(c *v1beta1.Cadence, latestCassandra Finalizers: []string{}, } - if len(c.Spec.DataCentres) == 0 { - return nil, models.ErrZeroDataCentres - } - slaTier := c.Spec.SLATier - privateNetwork := c.Spec.PrivateNetworkCluster + privateNetwork := c.Spec.PrivateNetwork pciCompliance := c.Spec.PCICompliance var twoFactorDelete []*v1beta1.TwoFactorDelete @@ -650,23 +635,13 @@ func (r *CadenceReconciler) newCassandraSpec(c *v1beta1.Cadence, latestCassandra var cassNodeSize, network string var cassNodesNumber, cassReplicationFactor int var cassPrivateIPBroadcastForDiscovery, cassPasswordAndUserAuth bool - for _, dc := range c.Spec.DataCentres { - for _, pp := range c.Spec.PackagedProvisioning { - cassNodeSize = pp.BundledCassandraSpec.NodeSize - network = pp.BundledCassandraSpec.Network - cassNodesNumber = pp.BundledCassandraSpec.NodesNumber - cassReplicationFactor = pp.BundledCassandraSpec.ReplicationFactor - cassPrivateIPBroadcastForDiscovery = pp.BundledCassandraSpec.PrivateIPBroadcastForDiscovery - cassPasswordAndUserAuth = pp.BundledCassandraSpec.PasswordAndUserAuth - - isCassNetworkOverlaps, err := dc.IsNetworkOverlaps(network) - if err != nil { - return nil, err - } - if isCassNetworkOverlaps { - return nil, models.ErrNetworkOverlaps - } - } + for _, pp := range c.Spec.PackagedProvisioning { + cassNodeSize = pp.BundledCassandraSpec.NodeSize + network = pp.BundledCassandraSpec.Network + cassNodesNumber = pp.BundledCassandraSpec.NodesNumber + cassReplicationFactor = pp.BundledCassandraSpec.ReplicationFactor + cassPrivateIPBroadcastForDiscovery = pp.BundledCassandraSpec.PrivateIPBroadcastForDiscovery + cassPasswordAndUserAuth = pp.BundledCassandraSpec.PasswordAndUserAuth } dcName := models.CassandraChildDCName @@ -689,6 +664,7 @@ func (r *CadenceReconciler) newCassandraSpec(c *v1beta1.Cadence, latestCassandra PrivateIPBroadcastForDiscovery: cassPrivateIPBroadcastForDiscovery, }, } + spec := v1beta1.CassandraSpec{ GenericClusterSpec: v1beta1.GenericClusterSpec{ Name: models.CassandraChildPrefix + c.Name, @@ -699,8 +675,8 @@ func (r *CadenceReconciler) newCassandraSpec(c *v1beta1.Cadence, latestCassandra }, DataCentres: cassandraDataCentres, PasswordAndUserAuth: cassPasswordAndUserAuth, - BundledUseOnly: true, PCICompliance: pciCompliance, + BundledUseOnly: true, } return &v1beta1.Cassandra{ @@ -710,8 +686,8 @@ func (r *CadenceReconciler) newCassandraSpec(c *v1beta1.Cadence, latestCassandra }, nil } -func (r *CadenceReconciler) startClusterStatusJob(c *v1beta1.Cadence) error { - job := r.newWatchStatusJob(c) +func (r *CadenceReconciler) startSyncJob(c *v1beta1.Cadence) error { + job := r.newSyncJob(c) err := r.Scheduler.ScheduleJob(c.GetJobID(scheduler.StatusChecker), scheduler.ClusterStatusInterval, job) if err != nil { @@ -721,8 +697,8 @@ func (r *CadenceReconciler) startClusterStatusJob(c *v1beta1.Cadence) error { return nil } -func (r *CadenceReconciler) newWatchStatusJob(c *v1beta1.Cadence) scheduler.Job { - l := log.Log.WithValues("component", "cadenceStatusClusterJob") +func (r *CadenceReconciler) newSyncJob(c *v1beta1.Cadence) scheduler.Job { + l := log.Log.WithValues("syncJob", c.GetJobID(scheduler.StatusChecker), "clusterID", c.Status.ID) return func() error { namespacedName := client.ObjectKeyFromObject(c) err := r.Get(context.Background(), namespacedName, c) @@ -739,7 +715,7 @@ func (r *CadenceReconciler) newWatchStatusJob(c *v1beta1.Cadence) scheduler.Job return err } - iData, err := r.API.GetCadence(c.Status.ID) + instaModel, err := r.API.GetCadence(c.Status.ID) if err != nil { if errors.Is(err, instaclustr.NotFound) { if c.DeletionTimestamp != nil { @@ -756,26 +732,16 @@ func (r *CadenceReconciler) newWatchStatusJob(c *v1beta1.Cadence) scheduler.Job return err } - iCadence, err := c.FromInstAPI(iData) - if err != nil { - l.Error(err, "Cannot convert Cadence cluster from the Instaclustr API", - "clusterID", c.Status.ID, - ) - return err - } + iCadence := &v1beta1.Cadence{} + iCadence.FromInstAPI(instaModel) - if !areStatusesEqual(&iCadence.Status.ClusterStatus, &c.Status.ClusterStatus) || - !areSecondaryCadenceTargetsEqual(c.Status.TargetSecondaryCadence, iCadence.Status.TargetSecondaryCadence) { - l.Info("Updating Cadence cluster status", - "new status", iCadence.Status.ClusterStatus, - "old status", c.Status.ClusterStatus, - ) + if !c.Status.Equals(&iCadence.Status) { + l.Info("Updating Cadence cluster status") - areDCsEqual := areDataCentresEqual(iCadence.Status.ClusterStatus.DataCentres, c.Status.ClusterStatus.DataCentres) + areDCsEqual := c.Status.DCsEqual(iCadence.Status.DataCentres) patch := c.NewPatch() - c.Status.ClusterStatus = iCadence.Status.ClusterStatus - c.Status.TargetSecondaryCadence = iCadence.Status.TargetSecondaryCadence + c.Status.FromInstAPI(instaModel) err = r.Status().Patch(context.Background(), c, patch) if err != nil { l.Error(err, "Cannot patch Cadence cluster", @@ -788,14 +754,14 @@ func (r *CadenceReconciler) newWatchStatusJob(c *v1beta1.Cadence) scheduler.Job if !areDCsEqual { var nodes []*v1beta1.Node - for _, dc := range iCadence.Status.ClusterStatus.DataCentres { + for _, dc := range iCadence.Status.DataCentres { nodes = append(nodes, dc.Nodes...) } err = exposeservice.Create(r.Client, c.Name, c.Namespace, - c.Spec.PrivateNetworkCluster, + c.Spec.PrivateNetwork, nodes, models.CadenceConnectionPort) if err != nil { @@ -804,22 +770,14 @@ func (r *CadenceReconciler) newWatchStatusJob(c *v1beta1.Cadence) scheduler.Job } } - equals := c.Spec.IsEqual(iCadence.Spec) - - if equals && c.Annotations[models.ExternalChangesAnnotation] == models.True { - patch := c.NewPatch() - delete(c.Annotations, models.ExternalChangesAnnotation) - err := r.Patch(context.Background(), c, patch) + if c.Annotations[models.ExternalChangesAnnotation] == models.True && c.IsSpecEqual(iCadence.Spec) { + err = reconcileExternalChanges(r.Client, r.EventRecorder, c) if err != nil { return err } - - r.EventRecorder.Event(c, models.Normal, models.ExternalChanges, - "External changes were automatically reconciled", - ) } else if c.Status.CurrentClusterOperationStatus == models.NoOperation && c.Annotations[models.ResourceStateAnnotation] != models.UpdatingEvent && - !equals { + !c.IsSpecEqual(iCadence.Spec) { l.Info(msgExternalChanges, "instaclustr data", iCadence.Spec.DataCentres, "k8s resource spec", c.Spec.DataCentres) @@ -834,7 +792,7 @@ func (r *CadenceReconciler) newWatchStatusJob(c *v1beta1.Cadence) scheduler.Job return err } - msgDiffSpecs, err := createSpecDifferenceMessage(c.Spec.DataCentres, iCadence.Spec.DataCentres) + msgDiffSpecs, err := createSpecDifferenceMessage(c.Spec, iCadence.Spec) if err != nil { l.Error(err, "Cannot create specification difference message", "instaclustr data", iCadence.Spec, "k8s resource spec", c.Spec) @@ -914,16 +872,6 @@ func (r *CadenceReconciler) newKafkaSpec(c *v1beta1.Cadence, latestKafkaVersion bundledKafkaSpec := c.Spec.PackagedProvisioning[0].BundledKafkaSpec kafkaNetwork := bundledKafkaSpec.Network - for _, cadenceDC := range c.Spec.DataCentres { - isKafkaNetworkOverlaps, err := cadenceDC.IsNetworkOverlaps(kafkaNetwork) - if err != nil { - return nil, err - } - if isKafkaNetworkOverlaps { - return nil, models.ErrNetworkOverlaps - } - } - kafkaNodeSize := bundledKafkaSpec.NodeSize kafkaNodesNumber := bundledKafkaSpec.NodesNumber dcName := models.KafkaChildDCName @@ -945,7 +893,7 @@ func (r *CadenceReconciler) newKafkaSpec(c *v1beta1.Cadence, latestKafkaVersion } slaTier := c.Spec.SLATier - privateClusterNetwork := c.Spec.PrivateNetworkCluster + privateClusterNetwork := c.Spec.PrivateNetwork pciCompliance := c.Spec.PCICompliance clientEncryption := c.Spec.DataCentres[0].ClientEncryption spec := v1beta1.KafkaSpec{ @@ -1000,7 +948,7 @@ func (r *CadenceReconciler) newOpenSearchSpec(c *v1beta1.Cadence, oldestOpenSear oNumberOfRacks := bundledOpenSearchSpec.NumberOfRacks slaTier := c.Spec.SLATier - privateClusterNetwork := c.Spec.PrivateNetworkCluster + privateClusterNetwork := c.Spec.PrivateNetwork pciCompliance := c.Spec.PCICompliance var twoFactorDelete []*v1beta1.TwoFactorDelete @@ -1014,14 +962,6 @@ func (r *CadenceReconciler) newOpenSearchSpec(c *v1beta1.Cadence, oldestOpenSear } osNetwork := bundledOpenSearchSpec.Network - isOsNetworkOverlaps, err := c.Spec.DataCentres[0].IsNetworkOverlaps(osNetwork) - if err != nil { - return nil, err - } - if isOsNetworkOverlaps { - return nil, models.ErrNetworkOverlaps - } - dcName := models.OpenSearchChildDCName dcRegion := c.Spec.DataCentres[0].Region cloudProvider := c.Spec.DataCentres[0].CloudProvider @@ -1119,16 +1059,6 @@ func (r *CadenceReconciler) deletePackagedResources( return nil } -func areSecondaryCadenceTargetsEqual(k8sTargets, iTargets []*v1beta1.TargetCadence) bool { - for _, iTarget := range iTargets { - for _, k8sTarget := range k8sTargets { - return *iTarget == *k8sTarget - } - } - - return len(iTargets) == len(k8sTargets) -} - // SetupWithManager sets up the controller with the Manager. func (r *CadenceReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). @@ -1189,7 +1119,7 @@ func (r *CadenceReconciler) reconcileMaintenanceEvents(ctx context.Context, c *v return err } - if !c.Status.AreMaintenanceEventStatusesEqual(iMEStatuses) { + if !c.Status.MaintenanceEventsEqual(iMEStatuses) { patch := c.NewPatch() c.Status.MaintenanceEvents = iMEStatuses err = r.Status().Patch(ctx, c, patch) diff --git a/controllers/clusters/datatest/cadence_v1beta1.yaml b/controllers/clusters/datatest/cadence_v1beta1.yaml index 808c22941..c910f00e3 100644 --- a/controllers/clusters/datatest/cadence_v1beta1.yaml +++ b/controllers/clusters/datatest/cadence_v1beta1.yaml @@ -52,7 +52,7 @@ spec: name: "testdc" # nodeSize: "CAD-PRD-m5ad.large-75" nodeSize: "cadence-test-node-size-1" - nodesNumber: 1 + numberOfNodes: 1 clientEncryption: false # privateLink: # - advertisedHostname: "cadence-sample-test.com" diff --git a/controllers/clusters/datatest/cadence_v1beta1_packaged.yaml b/controllers/clusters/datatest/cadence_v1beta1_packaged.yaml index dd7e1016c..6cd7aaff9 100644 --- a/controllers/clusters/datatest/cadence_v1beta1_packaged.yaml +++ b/controllers/clusters/datatest/cadence_v1beta1_packaged.yaml @@ -52,7 +52,7 @@ spec: name: "testdc" # nodeSize: "CAD-PRD-m5ad.large-75" nodeSize: "cadence-test-node-size-1" - nodesNumber: 1 + numberOfNodes: 1 clientEncryption: false # privateLink: # - advertisedHostname: "cadence-sample-test.com" diff --git a/pkg/instaclustr/client.go b/pkg/instaclustr/client.go index c9ce45b59..34a6896cd 100644 --- a/pkg/instaclustr/client.go +++ b/pkg/instaclustr/client.go @@ -2035,7 +2035,7 @@ func (c *Client) ResetPostgreSQLConfiguration(id, name string) error { return nil } -func (c *Client) GetCadence(id string) ([]byte, error) { +func (c *Client) GetCadence(id string) (*models.CadenceCluster, error) { url := c.serverHostname + CadenceEndpoint + id resp, err := c.DoRequest(url, http.MethodGet, nil) if err != nil { @@ -2056,7 +2056,13 @@ func (c *Client) GetCadence(id string) ([]byte, error) { return nil, fmt.Errorf("status code: %d, message: %s", resp.StatusCode, body) } - return body, nil + var instaModel models.CadenceCluster + err = json.Unmarshal(body, &instaModel) + if err != nil { + return nil, err + } + + return &instaModel, nil } func (c *Client) UpdatePostgreSQLDefaultUserPassword(id, password string) error { diff --git a/pkg/instaclustr/interfaces.go b/pkg/instaclustr/interfaces.go index 1ecb489a6..70f751bab 100644 --- a/pkg/instaclustr/interfaces.go +++ b/pkg/instaclustr/interfaces.go @@ -92,7 +92,7 @@ type API interface { CreatePostgreSQLConfiguration(id, name, value string) error UpdatePostgreSQLConfiguration(id, name, value string) error ResetPostgreSQLConfiguration(id, name string) error - GetCadence(id string) ([]byte, error) + GetCadence(id string) (*models.CadenceCluster, error) UpdatePostgreSQLDefaultUserPassword(id, password string) error ListClusters() ([]*models.ActiveClusters, error) CreateEncryptionKey(encryptionKeySpec any) (*clusterresourcesv1beta1.AWSEncryptionKeyStatus, error) diff --git a/pkg/instaclustr/mock/client.go b/pkg/instaclustr/mock/client.go index 22bd521c0..3860e9f2b 100644 --- a/pkg/instaclustr/mock/client.go +++ b/pkg/instaclustr/mock/client.go @@ -342,7 +342,7 @@ func (c *mockClient) ResetPostgreSQLConfiguration(id, name string) error { panic("ResetPostgreSQLConfiguration: is not implemented") } -func (c *mockClient) GetCadence(id string) ([]byte, error) { +func (c *mockClient) GetCadence(id string) (*models.CadenceCluster, error) { panic("GetCadence: is not implemented") } diff --git a/pkg/models/apiv2.go b/pkg/models/apiv2.go index ac3d0e429..727974763 100644 --- a/pkg/models/apiv2.go +++ b/pkg/models/apiv2.go @@ -63,6 +63,24 @@ type CloudProviderSettings struct { AzureSettings []*AzureSetting `json:"azureSettings,omitempty"` } +// HasAWSCloudProviderSettings indicates if the AWSSetting object is not zero. +// We need this because Instaclustr API returns an object of AWSSetting which is not empty, +// but the fields are filled with null values. +func (dc *CloudProviderSettings) HasAWSCloudProviderSettings() bool { + if dc == nil || len(dc.AWSSettings) == 0 { + return false + } + + zero := AWSSetting{} + for _, s := range dc.AWSSettings { + if s != nil && *s != zero { + return true + } + } + + return false +} + type AWSSetting struct { EBSEncryptionKey string `json:"ebsEncryptionKey,omitempty"` CustomVirtualNetworkID string `json:"customVirtualNetworkId,omitempty"` diff --git a/pkg/models/cadence_apiv2.go b/pkg/models/cadence_apiv2.go index c69c42c1c..a3d960329 100644 --- a/pkg/models/cadence_apiv2.go +++ b/pkg/models/cadence_apiv2.go @@ -19,36 +19,35 @@ package models const ( AWSAccessKeyID = "awsAccessKeyId" AWSSecretAccessKey = "awsSecretAccessKey" - - SharedProvisioningType = "SHARED" - PackagedProvisioningType = "PACKAGED" - StandardProvisioningType = "STANDARD" ) type CadenceCluster struct { - ClusterStatus `json:",inline"` - Name string `json:"name"` - CadenceVersion string `json:"cadenceVersion"` + GenericClusterFields `json:",inline"` + DataCentres []*CadenceDataCentre `json:"dataCentres"` SharedProvisioning []*CadenceSharedProvisioning `json:"sharedProvisioning,omitempty"` StandardProvisioning []*CadenceStandardProvisioning `json:"standardProvisioning,omitempty"` - PCIComplianceMode bool `json:"pciComplianceMode"` - TwoFactorDelete []*TwoFactorDelete `json:"twoFactorDelete,omitempty"` - UseCadenceWebAuth bool `json:"useCadenceWebAuth"` - PrivateNetworkCluster bool `json:"privateNetworkCluster"` - SLATier string `json:"slaTier"` AWSArchival []*AWSArchival `json:"awsArchival,omitempty"` - TargetPrimaryCadence []*TargetCadence `json:"targetPrimaryCadence,omitempty"` - TargetSecondaryCadence []*TargetCadence `json:"targetSecondaryCadence,omitempty"` + TwoFactorDelete []*TwoFactorDelete `json:"twoFactorDelete,omitempty"` + TargetPrimaryCadence []*CadenceDependencyTarget `json:"targetPrimaryCadence,omitempty"` + TargetSecondaryCadence []*CadenceDependencyTarget `json:"targetSecondaryCadence,omitempty"` ResizeSettings []*ResizeSettings `json:"resizeSettings,omitempty"` - Description string `json:"description,omitempty"` - UseHTTPAPI bool `json:"useHttpApi,omitempty"` + + CadenceVersion string `json:"cadenceVersion"` + UseHTTPAPI bool `json:"useHttpApi,omitempty"` + PCIComplianceMode bool `json:"pciComplianceMode"` + UseCadenceWebAuth bool `json:"useCadenceWebAuth"` } type CadenceDataCentre struct { - DataCentre `json:",inline"` - ClientToClusterEncryption bool `json:"clientToClusterEncryption"` - PrivateLink []*PrivateLink `json:"privateLink,omitempty"` + GenericDataCentreFields `json:",inline"` + + ClientToClusterEncryption bool `json:"clientToClusterEncryption"` + NumberOfNodes int `json:"numberOfNodes"` + NodeSize string `json:"nodeSize"` + + PrivateLink []*PrivateLink `json:"privateLink,omitempty"` + Nodes []*Node `json:"nodes"` } type CadenceSharedProvisioning struct { @@ -56,31 +55,16 @@ type CadenceSharedProvisioning struct { } type CadenceStandardProvisioning struct { - AdvancedVisibility []*AdvancedVisibility `json:"advancedVisibility,omitempty"` - TargetCassandra *TargetCassandra `json:"targetCassandra"` + AdvancedVisibility []*AdvancedVisibility `json:"advancedVisibility,omitempty"` + TargetCassandra *CadenceDependencyTarget `json:"targetCassandra"` } type AdvancedVisibility struct { - TargetKafka *TargetKafka `json:"targetKafka"` - TargetOpenSearch *TargetOpenSearch `json:"targetOpenSearch"` -} - -type TargetKafka struct { - DependencyCDCID string `json:"dependencyCdcId"` - DependencyVPCType string `json:"dependencyVpcType"` -} - -type TargetOpenSearch struct { - DependencyCDCID string `json:"dependencyCdcId"` - DependencyVPCType string `json:"dependencyVpcType"` -} - -type TargetCassandra struct { - DependencyCDCID string `json:"dependencyCdcId"` - DependencyVPCType string `json:"dependencyVpcType"` + TargetKafka *CadenceDependencyTarget `json:"targetKafka"` + TargetOpenSearch *CadenceDependencyTarget `json:"targetOpenSearch"` } -type TargetCadence struct { +type CadenceDependencyTarget struct { DependencyCDCID string `json:"dependencyCdcId"` DependencyVPCType string `json:"dependencyVpcType"` } @@ -93,5 +77,6 @@ type AWSArchival struct { } type CadenceClusterAPIUpdate struct { - DataCentres []*CadenceDataCentre `json:"dataCentres"` + DataCentres []*CadenceDataCentre `json:"dataCentres"` + ResizeSettings []*ResizeSettings `json:"resizeSettings,omitempty"` } diff --git a/pkg/models/errors.go b/pkg/models/errors.go index 59eaccf78..0fe345f36 100644 --- a/pkg/models/errors.go +++ b/pkg/models/errors.go @@ -24,7 +24,6 @@ var ( ErrZeroDataCentres = errors.New("cluster spec doesn't have data centres") ErrMoreThanOneKraft = errors.New("cluster spec does not support more than one kraft") ErrMoreThanThreeControllerNodeCount = errors.New("kraft does not support more than three controller nodes") - ErrNetworkOverlaps = errors.New("cluster network overlaps") ErrImmutableTwoFactorDelete = errors.New("twoFactorDelete field is immutable") ErrImmutableIngestNodes = errors.New("IngestNodes field is immutable") ErrImmutableClusterManagedNodes = errors.New("ClusterManagedNodes field is immutable")