From a738868f4d8cae1da28a20d8f79735615412fbf7 Mon Sep 17 00:00:00 2001 From: Jared Edwards Date: Fri, 17 Nov 2023 12:47:09 -0700 Subject: [PATCH] initial commit for adding machine type and count (#229) * initial commit for adding machine type and count * bailing - back to cleanup * update token values for structs and tokens * pulls in latest main and adjusts digitalocean physicals * cleanup and fix tokens --- docs/docs.go | 31 ++++++++++++-- docs/swagger.json | 31 ++++++++++++-- docs/swagger.yaml | 22 +++++++++- go.mod | 2 +- go.sum | 4 +- internal/controller/cluster.go | 2 + internal/controller/controller.go | 4 ++ internal/env/env.go | 44 ++++++++++---------- internal/environments/defaultEnvironments.go | 2 +- pkg/providerConfigs/detokenize.go | 6 ++- pkg/providerConfigs/types.go | 39 ++++++++--------- pkg/types/cluster.go | 15 ++++--- providers/aws/create.go | 1 - 13 files changed, 142 insertions(+), 61 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 7f090bb8..94bad53c 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -931,12 +931,17 @@ const docTemplate = `{ "type": "string" }, "token": { + "description": "DEPRECATED: please transition to APIToken", "type": "string" } } }, "types.Cluster": { "type": "object", + "required": [ + "node_count", + "node_type" + ], "properties": { "_id": { "type": "string" @@ -1091,6 +1096,12 @@ const docTemplate = `{ "last_condition": { "type": "string" }, + "node_count": { + "type": "string" + }, + "node_type": { + "type": "string" + }, "state_store_create_check": { "type": "boolean" }, @@ -1147,6 +1158,8 @@ const docTemplate = `{ "domain_name", "git_protocol", "git_provider", + "node_count", + "node_type", "type" ], "properties": { @@ -1227,6 +1240,12 @@ const docTemplate = `{ "google_auth": { "$ref": "#/definitions/types.GoogleAuth" }, + "node_count": { + "type": "string" + }, + "node_type": { + "type": "string" + }, "subdomain_name": { "type": "string" }, @@ -1310,9 +1329,15 @@ const docTemplate = `{ "types.Environment": { "type": "object", "properties": { + "_id": { + "type": "string" + }, "color": { "type": "string" }, + "creation_timestamp": { + "type": "string" + }, "description": { "type": "string" }, @@ -1616,12 +1641,12 @@ const docTemplate = `{ "instance_size": { "type": "string" }, - "machine_type": { - "type": "string" - }, "node_count": { "type": "integer" }, + "node_type": { + "type": "string" + }, "status": { "type": "string" } diff --git a/docs/swagger.json b/docs/swagger.json index bec69b62..14de17b2 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -925,12 +925,17 @@ "type": "string" }, "token": { + "description": "DEPRECATED: please transition to APIToken", "type": "string" } } }, "types.Cluster": { "type": "object", + "required": [ + "node_count", + "node_type" + ], "properties": { "_id": { "type": "string" @@ -1085,6 +1090,12 @@ "last_condition": { "type": "string" }, + "node_count": { + "type": "string" + }, + "node_type": { + "type": "string" + }, "state_store_create_check": { "type": "boolean" }, @@ -1141,6 +1152,8 @@ "domain_name", "git_protocol", "git_provider", + "node_count", + "node_type", "type" ], "properties": { @@ -1221,6 +1234,12 @@ "google_auth": { "$ref": "#/definitions/types.GoogleAuth" }, + "node_count": { + "type": "string" + }, + "node_type": { + "type": "string" + }, "subdomain_name": { "type": "string" }, @@ -1304,9 +1323,15 @@ "types.Environment": { "type": "object", "properties": { + "_id": { + "type": "string" + }, "color": { "type": "string" }, + "creation_timestamp": { + "type": "string" + }, "description": { "type": "string" }, @@ -1610,12 +1635,12 @@ "instance_size": { "type": "string" }, - "machine_type": { - "type": "string" - }, "node_count": { "type": "integer" }, + "node_type": { + "type": "string" + }, "status": { "type": "string" } diff --git a/docs/swagger.yaml b/docs/swagger.yaml index efb664bc..1c6edf83 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -36,6 +36,7 @@ definitions: origin_ca_issuer_key: type: string token: + description: 'DEPRECATED: please transition to APIToken' type: string type: object types.Cluster: @@ -142,6 +143,10 @@ definitions: type: string last_condition: type: string + node_count: + type: string + node_type: + type: string state_store_create_check: type: boolean state_store_credentials: @@ -172,6 +177,9 @@ definitions: items: $ref: '#/definitions/types.WorkloadCluster' type: array + required: + - node_count + - node_type type: object types.ClusterDefinition: properties: @@ -228,6 +236,10 @@ definitions: type: string google_auth: $ref: '#/definitions/types.GoogleAuth' + node_count: + type: string + node_type: + type: string subdomain_name: type: string type: @@ -245,6 +257,8 @@ definitions: - domain_name - git_protocol - git_provider + - node_count + - node_type - type type: object types.ClusterServiceList: @@ -291,8 +305,12 @@ definitions: type: object types.Environment: properties: + _id: + type: string color: type: string + creation_timestamp: + type: string description: type: string name: @@ -490,10 +508,10 @@ definitions: $ref: '#/definitions/types.GitAuth' instance_size: type: string - machine_type: - type: string node_count: type: integer + node_type: + type: string status: type: string type: object diff --git a/go.mod b/go.mod index c4d4ee87..84dadaf5 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/hashicorp/vault/api v1.9.0 github.com/joho/godotenv v1.5.1 github.com/kubefirst/metrics-client v0.2.8 - github.com/kubefirst/runtime v0.3.27 + github.com/kubefirst/runtime v0.3.30 github.com/minio/minio-go/v7 v7.0.49 github.com/otiai10/copy v1.7.0 github.com/rs/zerolog v1.29.1 diff --git a/go.sum b/go.sum index c1bd733a..37752cb7 100644 --- a/go.sum +++ b/go.sum @@ -740,8 +740,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubefirst/metrics-client v0.2.8 h1:JfaeiBifZD/DpyYW2QVHcrhH/KWA98OmM+7c7M12qMc= github.com/kubefirst/metrics-client v0.2.8/go.mod h1:GR7wsMcyYhd+EU67PeuMCBYFE6OJ7P/j5OI5BLOoRMc= -github.com/kubefirst/runtime v0.3.27 h1:U8u+ywqNbbSra8W4jZe+mHrkaZ0hE6JaHSWy3XQ/8GU= -github.com/kubefirst/runtime v0.3.27/go.mod h1:9egAgF5cF594mIzra5nTB1kfE57Hr/Nvl79tjLsAWCQ= +github.com/kubefirst/runtime v0.3.30 h1:faMT6w//vyt7v4qeBwh74SCNDoIv/+AWTWOnG9XyzOc= +github.com/kubefirst/runtime v0.3.30/go.mod h1:9egAgF5cF594mIzra5nTB1kfE57Hr/Nvl79tjLsAWCQ= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= diff --git a/internal/controller/cluster.go b/internal/controller/cluster.go index 2f4a1251..860c1536 100644 --- a/internal/controller/cluster.go +++ b/internal/controller/cluster.go @@ -172,6 +172,8 @@ func (clctrl *ClusterController) CreateTokens(kind string) interface{} { SubdomainName: clctrl.SubdomainName, KubefirstStateStoreBucket: clctrl.KubefirstStateStoreBucketName, KubefirstTeam: clctrl.KubefirstTeam, + NodeType: clctrl.NodeType, + NodeCount: clctrl.NodeCount, KubefirstVersion: env.KubefirstVersion, Kubeconfig: clctrl.ProviderConfig.Kubeconfig, //AWS KubeconfigPath: clctrl.ProviderConfig.Kubeconfig, //Not AWS diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 9624cf66..e9106b05 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -59,6 +59,8 @@ type ClusterController struct { GoogleAuth pkgtypes.GoogleAuth AwsAccessKeyID string AwsSecretAccessKey string + NodeType string + NodeCount int // configs ProviderConfig providerConfigs.ProviderConfig @@ -206,6 +208,8 @@ func (clctrl *ClusterController) InitController(def *pkgtypes.ClusterDefinition) clctrl.KubefirstStateStoreBucketName = fmt.Sprintf("k1-state-store-%s-%s", clctrl.ClusterName, clusterID) clctrl.KubefirstArtifactsBucketName = fmt.Sprintf("k1-artifacts-%s-%s", clctrl.ClusterName, clusterID) + clctrl.NodeType = def.NodeType + clctrl.NodeCount = def.NodeCount clctrl.KubefirstTeam = env.KubefirstTeam diff --git a/internal/env/env.go b/internal/env/env.go index c2946fb4..d061941a 100644 --- a/internal/env/env.go +++ b/internal/env/env.go @@ -7,29 +7,29 @@ import ( ) type Env struct { - ServerPort int `env:"SERVER_PORT" envDefault:"8081"` - K1AccessToken string `env:"K1_ACCESS_TOKEN"` - MongoDBHost string `env:"MONGODB_HOST,notEmpty"` - MongoDBHostType string `env:"MONGODB_HOST_TYPE,notEmpty"` - MongoDBUsername string `env:"MONGODB_USERNAME,notEmpty"` - MongoDBPassword string `env:"MONGODB_PASSWORD,notEmpty"` - KubefirstVersion string `env:"KUBEFIRST_VERSION" envDefault:"development"` - CloudProvider string `env:"CLOUD_PROVIDER"` - ClusterId string `env:"CLUSTER_ID"` - ClusterType string `env:"CLUSTER_TYPE"` - DomainName string `env:"DOMAIN_NAME"` - GitProvider string `env:"GIT_PROVIDER"` - InstallMethod string `env:"INSTALL_METHOD"` - KubefirstTeam string `env:"KUBEFIRST_TEAM" envDefault:"undefined"` - KubefirstTeamInfo string `env:"KUBEFIRST_TEAM_INFO"` - AWSRegion string `env:"AWS_REGION"` - AWSProfile string `env:"AWS_PROFILE"` - IsClusterZero bool `env:"IS_CLUSTER_ZERO"` - InCluster bool `env:"IN_CLUSTER" envDefault:"false"` - EnterpriseApiUrl string `env:"ENTERPRISE_API_URL"` + ServerPort int `env:"SERVER_PORT" envDefault:"8081"` + K1AccessToken string `env:"K1_ACCESS_TOKEN"` + MongoDBHost string `env:"MONGODB_HOST,notEmpty"` + MongoDBHostType string `env:"MONGODB_HOST_TYPE,notEmpty"` + MongoDBUsername string `env:"MONGODB_USERNAME,notEmpty"` + MongoDBPassword string `env:"MONGODB_PASSWORD,notEmpty"` + KubefirstVersion string `env:"KUBEFIRST_VERSION" envDefault:"development"` + CloudProvider string `env:"CLOUD_PROVIDER"` + ClusterId string `env:"CLUSTER_ID"` + ClusterType string `env:"CLUSTER_TYPE"` + DomainName string `env:"DOMAIN_NAME"` + GitProvider string `env:"GIT_PROVIDER"` + InstallMethod string `env:"INSTALL_METHOD"` + KubefirstTeam string `env:"KUBEFIRST_TEAM" envDefault:"undefined"` + KubefirstTeamInfo string `env:"KUBEFIRST_TEAM_INFO"` + AWSRegion string `env:"AWS_REGION"` + AWSProfile string `env:"AWS_PROFILE"` + IsClusterZero bool `env:"IS_CLUSTER_ZERO"` + InCluster bool `env:"IN_CLUSTER" envDefault:"false"` + EnterpriseApiUrl string `env:"ENTERPRISE_API_URL"` } -func GetEnv()(Env,error) { +func GetEnv() (Env, error) { envError := godotenv.Load(".env") if envError != nil { @@ -43,4 +43,4 @@ func GetEnv()(Env,error) { } return environment, nil -} \ No newline at end of file +} diff --git a/internal/environments/defaultEnvironments.go b/internal/environments/defaultEnvironments.go index 6b24542d..6cbe0e2b 100644 --- a/internal/environments/defaultEnvironments.go +++ b/internal/environments/defaultEnvironments.go @@ -60,7 +60,7 @@ func CreateDefaultEnvironments(mgmtCluster types.Cluster) error { }, GitAuth: mgmtCluster.GitAuth, InstanceSize: "", // left up to terraform - MachineType: "", //left up to terraform + NodeType: "", //left up to terraform NodeCount: 3, //defaulted here } diff --git a/pkg/providerConfigs/detokenize.go b/pkg/providerConfigs/detokenize.go index d9e9e465..104c683b 100644 --- a/pkg/providerConfigs/detokenize.go +++ b/pkg/providerConfigs/detokenize.go @@ -81,6 +81,10 @@ func detokenizeGitops(path string, tokens *GitopsDirectoryValues, gitProtocol st newContents = strings.Replace(newContents, "", os.Getenv("KUBEFIRST_TEAM_INFO"), -1) newContents = strings.Replace(newContents, "", tokens.KubefirstVersion, -1) newContents = strings.Replace(newContents, "", tokens.StateStoreBucketHostname, -1) + newContents = strings.Replace(newContents, "", "git::https:////terraform//modules/workload-cluster?ref=main", -1) + + newContents = strings.Replace(newContents, "", tokens.NodeType, -1) + newContents = strings.Replace(newContents, "", fmt.Sprint(tokens.NodeCount), -1) // AWS newContents = strings.Replace(newContents, "", tokens.AwsAccountID, -1) @@ -111,7 +115,7 @@ func detokenizeGitops(path string, tokens *GitopsDirectoryValues, gitProtocol st newContents = strings.Replace(newContents, "", tokens.GitRunner, -1) newContents = strings.Replace(newContents, "", tokens.GitRunnerDescription, -1) newContents = strings.Replace(newContents, "", tokens.GitRunnerNS, -1) - newContents = strings.Replace(newContents, "", tokens.GitURL, -1) + newContents = strings.Replace(newContents, "", tokens.GitURL, -1) // remove // GitHub newContents = strings.Replace(newContents, "", tokens.GitHubHost, -1) diff --git a/pkg/providerConfigs/types.go b/pkg/providerConfigs/types.go index 9d401f6f..9a6d0f0f 100644 --- a/pkg/providerConfigs/types.go +++ b/pkg/providerConfigs/types.go @@ -7,25 +7,26 @@ See the LICENSE file for more details. package providerConfigs type GitopsDirectoryValues struct { - AlertsEmail string - AtlantisAllowList string - CloudProvider string - CloudRegion string - ClusterId string - ClusterName string - ClusterType string - ContainerRegistryURL string - DomainName string - SubdomainName string - DNSProvider string - Kubeconfig string - KubeconfigPath string - KubefirstArtifactsBucket string - KubefirstStateStoreBucket string - KubefirstTeam string - KubefirstVersion string - StateStoreBucketHostname string - + AlertsEmail string + AtlantisAllowList string + CloudProvider string + CloudRegion string + ClusterId string + ClusterName string + ClusterType string + ContainerRegistryURL string + DomainName string + SubdomainName string + DNSProvider string + Kubeconfig string + KubeconfigPath string + KubefirstArtifactsBucket string + KubefirstStateStoreBucket string + KubefirstTeam string + KubefirstVersion string + StateStoreBucketHostname string + NodeType string + NodeCount int ArgoCDIngressURL string ArgoCDIngressNoHTTPSURL string ArgoWorkflowsIngressURL string diff --git a/pkg/types/cluster.go b/pkg/types/cluster.go index afa802c1..a096b8e9 100644 --- a/pkg/types/cluster.go +++ b/pkg/types/cluster.go @@ -23,7 +23,8 @@ type ClusterDefinition struct { DnsProvider string `json:"dns_provider,omitempty" binding:"required"` Type string `json:"type" binding:"required,oneof=mgmt workload"` ForceDestroy bool `bson:"force_destroy,omitempty" json:"force_destroy,omitempty"` - + NodeType string `json:"node_type" binding:"required"` + NodeCount int `json:"node_count" binding:"required"` //Git GitopsTemplateURL string `json:"gitops_template_url"` @@ -85,6 +86,8 @@ type Cluster struct { AtlantisWebhookSecret string `bson:"atlantis_webhook_secret" json:"atlantis_webhook_secret"` AtlantisWebhookURL string `bson:"atlantis_webhook_url" json:"atlantis_webhook_url"` KubefirstTeam string `bson:"kubefirst_team" json:"kubefirst_team"` + NodeType string `bson:"node_type" json:"node_type" binding:"required"` + NodeCount int `bson:"node_count" json:"node_count" binding:"required"` StateStoreCredentials StateStoreCredentials `bson:"state_store_credentials,omitempty" json:"state_store_credentials,omitempty"` StateStoreDetails StateStoreDetails `bson:"state_store_details,omitempty" json:"state_store_details,omitempty"` @@ -150,10 +153,10 @@ type ProxyImportRequest struct { type Environment struct { ID primitive.ObjectID `bson:"_id" json:"_id"` - Name string `bson:"name" json:"name"` - Color string `bson:"color" json:"color"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - CreationTimestamp string `bson:"creation_timestamp" json:"creation_timestamp"` + Name string `bson:"name" json:"name"` + Color string `bson:"color" json:"color"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + CreationTimestamp string `bson:"creation_timestamp" json:"creation_timestamp"` } type WorkloadCluster struct { @@ -169,7 +172,7 @@ type WorkloadCluster struct { Environment Environment `bson:"environment,omitempty" json:"environment,omitempty"` GitAuth GitAuth `bson:"git_auth,omitempty" json:"git_auth,omitempty"` InstanceSize string `bson:"instance_size,omitempty" json:"instance_size,omitempty"` - MachineType string `bson:"machine_type,omitempty" json:"machine_type,omitempty"` + NodeType string `bson:"node_type,omitempty" json:"node_type,omitempty"` NodeCount int `bson:"node_count,omitempty" json:"node_count,omitempty"` Status string `bson:"status,omitempty" json:"status,omitempty"` } diff --git a/providers/aws/create.go b/providers/aws/create.go index 6dd6c5c5..d1e25a25 100644 --- a/providers/aws/create.go +++ b/providers/aws/create.go @@ -235,7 +235,6 @@ func CreateAWSCluster(definition *pkgtypes.ClusterDefinition) error { return err } - log.Info("cluster creation complete") //* export and import cluster