From ff437cee488c2435c8331d1aab9459d324710a2c Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Thu, 21 Nov 2024 07:51:28 -0600 Subject: [PATCH 1/9] DWX-19297 Extend hive virtual warehouse properties Add all available configuration parameters to the hive VW. --- docs/resources/dw_vw_hive.md | 45 ++++++ .../dw/virtualwarehouse/hive/model_hive_vw.go | 130 +++++++++++++++++- .../virtualwarehouse/hive/resource_hive_vw.go | 7 +- .../virtualwarehouse/hive/schema_hive_vw.go | 90 ++++++++++++ utils/utils.go | 12 +- 5 files changed, 269 insertions(+), 15 deletions(-) diff --git a/docs/resources/dw_vw_hive.md b/docs/resources/dw_vw_hive.md index deb6a33f..e673f2a8 100644 --- a/docs/resources/dw_vw_hive.md +++ b/docs/resources/dw_vw_hive.md @@ -36,11 +36,19 @@ resource "cdp_dw_vw_hive" "example" { - `cluster_id` (String) The id of the CDW Cluster which the Hive Virtual Warehouse is attached to. - `database_catalog_id` (String) The id of the Database Catalog which the Hive Virtual Warehouse is attached to. +- `image_version` (String) The version of the Hive Virtual Warehouse image. - `name` (String) The name of the Hive Virtual Warehouse. ### Optional +- `autoscaling` (Attributes) Autoscaling related configuration options that could specify various values that will be used during CDW resource creation. (see [below for nested schema](#nestedatt--autoscaling)) +- `aws_options` (Attributes) AWS related configuration options that could specify various values that will be used during CDW resource creation. (see [below for nested schema](#nestedatt--aws_options)) +- `enable_sso` (Boolean) Enable SSO for the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. +- `ldap_groups` (List of String) LDAP group names to be enabled for auth. +- `node_count` (Number) Nodes per compute cluster. If specified, forces ‘template’ to be ‘custom’. +- `platform_jwt_auth` (Boolean) Value of ‘true’ automatically configures the Virtual Warehouse to support JWTs issued by the CDP JWT token provider. Value of ‘false’ does not enable JWT auth on the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. - `polling_options` (Attributes) Polling related configuration options that could specify various values that will be used during CDP resource creation. (see [below for nested schema](#nestedatt--polling_options)) +- `query_isolation_options` (Attributes) Query isolation related configuration options. (see [below for nested schema](#nestedatt--query_isolation_options)) ### Read-Only @@ -48,6 +56,32 @@ resource "cdp_dw_vw_hive" "example" { - `last_updated` (String) Timestamp of the last Terraform update of the order. - `status` (String) The status of the database catalog. + +### Nested Schema for `autoscaling` + +Required: + +- `auto_suspend_timeout_seconds` (Number) The time in seconds after which the compute group should be suspended. +- `disable_auto_suspend` (Boolean) Boolean value that specifies if auto-suspend should be disabled. +- `max_clusters` (Number) Maximum number of available compute groups. +- `min_clusters` (Number) Minimum number of available compute groups. + +Optional: + +- `hive_desired_free_capacity` (Number) Set Desired free capacity. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided. +- `hive_scale_wait_time_seconds` (Number) Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided. + + + +### Nested Schema for `aws_options` + +Optional: + +- `availability_zone` (String) This feature works only for AWS cluster type. An availability zone to host compute instances. If not specified, defaults to a randomly selected availability zone inferred from available subnets. +- `ebs_llap_spill_gb` (Number) This feature works only for AWS cluster type. The size of the EBS volume in GB to be used for LLAP spill storage. If not specified, defaults to no extra spill disk. +- `tags` (List of String) This feature works only for AWS cluster type. Tags to be applied to the underlying compute nodes. + + ### Nested Schema for `polling_options` @@ -56,3 +90,14 @@ Optional: - `async` (Boolean) Boolean value that specifies if Terraform should wait for resource creation/deletion. - `call_failure_threshold` (Number) Threshold value that specifies how many times should a single call failure happen before giving up the polling. - `polling_timeout` (Number) Timeout value in minutes that specifies for how long should the polling go for resource creation/deletion. + + + +### Nested Schema for `query_isolation_options` + +Optional: + +- `max_nodes_per_query` (Number) Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. +- `max_queries` (Number) Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. + + diff --git a/resources/dw/virtualwarehouse/hive/model_hive_vw.go b/resources/dw/virtualwarehouse/hive/model_hive_vw.go index 2b6d1485..93f1c2a4 100644 --- a/resources/dw/virtualwarehouse/hive/model_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/model_hive_vw.go @@ -13,19 +13,135 @@ package hive import ( "github.com/hashicorp/terraform-plugin-framework/types" + "github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/dw/models" "github.com/cloudera/terraform-provider-cdp/utils" ) +type autoscaling struct { + MinClusters types.Int64 `tfsdk:"min_clusters"` + MaxClusters types.Int64 `tfsdk:"max_clusters"` + DisableAutoSuspend types.Bool `tfsdk:"disable_auto_suspend"` + AutoSuspendTimeoutSeconds types.Int64 `tfsdk:"auto_suspend_timeout_seconds"` + HiveScaleWaitTimeSeconds types.Int64 `tfsdk:"hive_scale_wait_time_seconds"` + HiveDesiredFreeCapacity types.Int64 `tfsdk:"hive_desired_free_capacity"` +} + +type awsOptions struct { + AvailabilityZone types.String `tfsdk:"availability_zone"` + EbsLLAPSpillGb types.Int64 `tfsdk:"ebs_llap_spill_gb"` + Tags *[]resourceTags `tfsdk:"tags"` +} + +type queryIsolationOptions struct { + MaxQueries types.Int64 `tfsdk:"max_queries"` + MaxNodesPerQuery types.Int64 `tfsdk:"max_nodes_per_query"` +} + +type resourceTags struct { + Key types.String `tfsdk:"key"` + Value types.String `tfsdk:"value"` +} + type resourceModel struct { - ID types.String `tfsdk:"id"` - ClusterID types.String `tfsdk:"cluster_id"` - DatabaseCatalogID types.String `tfsdk:"database_catalog_id"` - Name types.String `tfsdk:"name"` - LastUpdated types.String `tfsdk:"last_updated"` - Status types.String `tfsdk:"status"` - PollingOptions *utils.PollingOptions `tfsdk:"polling_options"` + ID types.String `tfsdk:"id"` + ClusterID types.String `tfsdk:"cluster_id"` + DatabaseCatalogID types.String `tfsdk:"database_catalog_id"` + Name types.String `tfsdk:"name"` + ImageVersion types.String `tfsdk:"image_version"` + NodeCount types.Int64 `tfsdk:"node_count"` + PlatformJwtAuth types.Bool `tfsdk:"platform_jwt_auth"` + LdapGroups types.List `tfsdk:"ldap_groups"` + EnableSSO types.Bool `tfsdk:"enable_sso"` + Autoscaling *autoscaling `tfsdk:"autoscaling"` + AwsOptions *awsOptions `tfsdk:"aws_options"` + QueryIsolationOptions *queryIsolationOptions `tfsdk:"query_isolation_options"` + LastUpdated types.String `tfsdk:"last_updated"` + Status types.String `tfsdk:"status"` + PollingOptions *utils.PollingOptions `tfsdk:"polling_options"` } func (p *resourceModel) GetPollingOptions() *utils.PollingOptions { return p.PollingOptions } + +func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { + return &models.CreateVwRequest{ + ClusterID: p.ClusterID.ValueStringPointer(), + DbcID: p.DatabaseCatalogID.ValueStringPointer(), + EbsLLAPSpillGB: p.getEbsLLAPSpillGB(), + HiveServerHaMode: nil, // Private Cloud only option + ImageVersion: p.ImageVersion.String(), + Name: p.Name.ValueStringPointer(), + NodeCount: utils.Int64To32(p.NodeCount), + PlatformJwtAuth: p.PlatformJwtAuth.ValueBoolPointer(), + QueryIsolationOptions: p.getQueryIsolationOptions(), + Autoscaling: p.getAutoscaling(), + AvailabilityZone: p.getAvailabilityZone(), + Tags: p.getTags(), + Config: p.getServiceConfig(), + } +} + +func (p *resourceModel) getServiceConfig() *models.ServiceConfigReq { + return &models.ServiceConfigReq{ + ApplicationConfigs: nil, + CommonConfigs: nil, + EnableSSO: p.EnableSSO.ValueBool(), + LdapGroups: utils.FromListValueToStringList(p.LdapGroups), + } +} + +func (p *resourceModel) getTags() []*models.TagRequest { + if p.AwsOptions.Tags == nil { + return nil + } + tags := make([]*models.TagRequest, len(*p.AwsOptions.Tags)) + for _, tag := range *p.AwsOptions.Tags { + if tag.Key.IsNull() || tag.Value.IsNull() { + continue + } + tags = append(tags, &models.TagRequest{ + Key: tag.Key.ValueStringPointer(), + Value: tag.Value.ValueStringPointer(), + }) + } + return tags +} + +func (p *resourceModel) getQueryIsolationOptions() *models.QueryIsolationOptionsRequest { + if p.QueryIsolationOptions == nil { + return nil + } + return &models.QueryIsolationOptionsRequest{ + MaxQueries: utils.Int64To32(p.QueryIsolationOptions.MaxQueries), + MaxNodesPerQuery: utils.Int64To32(p.QueryIsolationOptions.MaxNodesPerQuery), + } +} + +func (p *resourceModel) getAvailabilityZone() string { + if p.AwsOptions == nil { + return "" + } + return p.AwsOptions.AvailabilityZone.ValueString() +} + +func (p *resourceModel) getEbsLLAPSpillGB() int32 { + if p.AwsOptions == nil { + return 0 + } + return utils.Int64To32(p.AwsOptions.EbsLLAPSpillGb) +} + +func (p *resourceModel) getAutoscaling() *models.AutoscalingOptionsCreateRequest { + if p.Autoscaling == nil { + return nil + } + return &models.AutoscalingOptionsCreateRequest{ + MinClusters: utils.Int64To32Pointer(p.Autoscaling.MinClusters), + MaxClusters: utils.Int64To32Pointer(p.Autoscaling.MaxClusters), + DisableAutoSuspend: p.Autoscaling.DisableAutoSuspend.ValueBool(), + AutoSuspendTimeoutSeconds: utils.Int64To32(p.Autoscaling.AutoSuspendTimeoutSeconds), + HiveScaleWaitTimeSeconds: utils.Int64To32(p.Autoscaling.HiveScaleWaitTimeSeconds), + HiveDesiredFreeCapacity: utils.Int64To32(p.Autoscaling.HiveDesiredFreeCapacity), + } +} diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go index 1b18bd2d..a553ea4d 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go @@ -69,12 +69,7 @@ func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, r // Generate API request body from plan vw := operations.NewCreateVwParamsWithContext(ctx). - WithInput(&models.CreateVwRequest{ - Name: plan.Name.ValueStringPointer(), - ClusterID: plan.ClusterID.ValueStringPointer(), - DbcID: plan.DatabaseCatalogID.ValueStringPointer(), - VwType: models.VwTypeHive.Pointer(), - }) + WithInput(plan.convertToCreateVwRequest()) // Create new virtual warehouse response, err := r.client.Dw.Operations.CreateVw(vw) diff --git a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go index 54babd09..a49ac436 100644 --- a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/terraform-plugin-framework/resource/schema/int64default" "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" + "github.com/hashicorp/terraform-plugin-framework/types" ) var hiveSchema = schema.Schema{ @@ -46,6 +47,95 @@ var hiveSchema = schema.Schema{ Required: true, MarkdownDescription: "The name of the Hive Virtual Warehouse.", }, + "image_version": schema.StringAttribute{ + Required: true, + MarkdownDescription: "The version of the Hive Virtual Warehouse image.", + }, + "node_count": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Nodes per compute cluster. If specified, forces ‘template’ to be ‘custom’.", + }, + "platform_jwt_auth": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Value of ‘true’ automatically configures the Virtual Warehouse to support JWTs issued by the CDP JWT token provider. Value of ‘false’ does not enable JWT auth on the Virtual Warehouse. If this field is not specified, it defaults to ‘false’.", + }, + "ldap_groups": schema.ListAttribute{ + Optional: true, + ElementType: types.StringType, + MarkdownDescription: "LDAP group names to be enabled for auth.", + }, + "enable_sso": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Enable SSO for the Virtual Warehouse. If this field is not specified, it defaults to ‘false’.", + }, + "autoscaling": schema.SingleNestedAttribute{ + MarkdownDescription: "Autoscaling related configuration options that could specify various values that will be used during CDW resource creation.", + Optional: true, + Attributes: map[string]schema.Attribute{ + "min_clusters": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Minimum number of available compute groups.", + }, + "max_clusters": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Maximum number of available compute groups.", + }, + "disable_auto_suspend": schema.BoolAttribute{ + Required: true, + MarkdownDescription: "Boolean value that specifies if auto-suspend should be disabled.", + }, + "auto_suspend_timeout_seconds": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "The time in seconds after which the compute group should be suspended.", + }, + "hive_scale_wait_time_seconds": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", + }, + "hive_desired_free_capacity": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set Desired free capacity. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", + }, + }, + }, + "aws_options": schema.SingleNestedAttribute{ + MarkdownDescription: "AWS related configuration options that could specify various values that will be used during CDW resource creation.", + Optional: true, + Attributes: map[string]schema.Attribute{ + "availability_zone": schema.StringAttribute{ + Optional: true, + Computed: true, + MarkdownDescription: "This feature works only for AWS cluster type. An availability zone to host compute instances. If not specified, defaults to a randomly selected availability zone inferred from available subnets.", + }, + "ebs_llap_spill_gb": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "This feature works only for AWS cluster type. The size of the EBS volume in GB to be used for LLAP spill storage. If not specified, defaults to no extra spill disk.", + }, + "tags": schema.ListAttribute{ + Optional: true, + ElementType: types.StringType, + MarkdownDescription: "This feature works only for AWS cluster type. Tags to be applied to the underlying compute nodes.", + }, + }, + }, + "query_isolation_options": schema.SingleNestedAttribute{ + MarkdownDescription: "Query isolation related configuration options.", + Optional: true, + Attributes: map[string]schema.Attribute{ + "max_queries": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", + }, + "max_nodes_per_query": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", + }, + }, + }, "last_updated": schema.StringAttribute{ Computed: true, MarkdownDescription: "Timestamp of the last Terraform update of the order.", diff --git a/utils/utils.go b/utils/utils.go index 8c489815..2f33d979 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -13,6 +13,7 @@ package utils import ( "context" "fmt" + "math" "time" "github.com/hashicorp/terraform-plugin-framework/datasource" @@ -133,9 +134,16 @@ func FromMapValueToStringMap(tl types.Map) map[string]string { } func Int64To32Pointer(in types.Int64) *int32 { + n32 := Int64To32(in) + return &n32 +} + +func Int64To32(in types.Int64) int32 { n64 := in.ValueInt64() - var n2 = int32(n64) - return &n2 + if n64 >= math.MinInt32 && n64 <= math.MaxInt32 { + return int32(n64) + } + panic(fmt.Sprintf("int64 value %d is out of range for int32", n64)) } type HasPollingOptions interface { From 7212597cf60538f666fa0db1c7476b55faa35c04 Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Wed, 4 Dec 2024 20:50:58 -0600 Subject: [PATCH 2/9] Add output parameters The jdbc url and other endpoint parameters are calculated during virtual warehouse creation. These are output parameters of a virtual warehouse. --- .../dw/virtualwarehouse/hive/model_hive_vw.go | 52 ++-- .../virtualwarehouse/hive/resource_hive_vw.go | 7 + .../hive/resource_hive_vw_acc_test.go | 14 + .../hive/resource_hive_vw_test.go | 256 ++++++++++++++++-- .../virtualwarehouse/hive/schema_hive_vw.go | 37 ++- 5 files changed, 318 insertions(+), 48 deletions(-) diff --git a/resources/dw/virtualwarehouse/hive/model_hive_vw.go b/resources/dw/virtualwarehouse/hive/model_hive_vw.go index 93f1c2a4..8fb770e4 100644 --- a/resources/dw/virtualwarehouse/hive/model_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/model_hive_vw.go @@ -27,9 +27,9 @@ type autoscaling struct { } type awsOptions struct { - AvailabilityZone types.String `tfsdk:"availability_zone"` - EbsLLAPSpillGb types.Int64 `tfsdk:"ebs_llap_spill_gb"` - Tags *[]resourceTags `tfsdk:"tags"` + AvailabilityZone types.String `tfsdk:"availability_zone"` + EbsLLAPSpillGb types.Int64 `tfsdk:"ebs_llap_spill_gb"` + Tags types.Map `tfsdk:"tags"` } type queryIsolationOptions struct { @@ -37,11 +37,6 @@ type queryIsolationOptions struct { MaxNodesPerQuery types.Int64 `tfsdk:"max_nodes_per_query"` } -type resourceTags struct { - Key types.String `tfsdk:"key"` - Value types.String `tfsdk:"value"` -} - type resourceModel struct { ID types.String `tfsdk:"id"` ClusterID types.String `tfsdk:"cluster_id"` @@ -52,6 +47,12 @@ type resourceModel struct { PlatformJwtAuth types.Bool `tfsdk:"platform_jwt_auth"` LdapGroups types.List `tfsdk:"ldap_groups"` EnableSSO types.Bool `tfsdk:"enable_sso"` + Compactor types.Bool `tfsdk:"compactor"` + JdbcUrl types.String `tfsdk:"jdbc_url"` + KerberosJdbcUrl types.String `tfsdk:"kerberos_jdbc_url"` + HueUrl types.String `tfsdk:"hue_url"` + JwtConnectionString types.String `tfsdk:"jwt_connection_string"` + JwtTokenGenUrl types.String `tfsdk:"jwt_token_gen_url"` Autoscaling *autoscaling `tfsdk:"autoscaling"` AwsOptions *awsOptions `tfsdk:"aws_options"` QueryIsolationOptions *queryIsolationOptions `tfsdk:"query_isolation_options"` @@ -65,21 +66,29 @@ func (p *resourceModel) GetPollingOptions() *utils.PollingOptions { } func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { + vwType := models.VwType("hive") return &models.CreateVwRequest{ - ClusterID: p.ClusterID.ValueStringPointer(), - DbcID: p.DatabaseCatalogID.ValueStringPointer(), - EbsLLAPSpillGB: p.getEbsLLAPSpillGB(), - HiveServerHaMode: nil, // Private Cloud only option - ImageVersion: p.ImageVersion.String(), + ClusterID: p.ClusterID.ValueStringPointer(), + DbcID: p.DatabaseCatalogID.ValueStringPointer(), + EbsLLAPSpillGB: p.getEbsLLAPSpillGB(), + //ImageVersion: p.getImageVersion(), Name: p.Name.ValueStringPointer(), NodeCount: utils.Int64To32(p.NodeCount), PlatformJwtAuth: p.PlatformJwtAuth.ValueBoolPointer(), QueryIsolationOptions: p.getQueryIsolationOptions(), Autoscaling: p.getAutoscaling(), AvailabilityZone: p.getAvailabilityZone(), - Tags: p.getTags(), - Config: p.getServiceConfig(), + //Tags: p.getTags(), + Config: p.getServiceConfig(), + VwType: &vwType, + } +} + +func (p *resourceModel) getImageVersion() string { + if p.ImageVersion.IsNull() || p.ImageVersion.String() == "unknown" { + return "" } + return p.ImageVersion.String() } func (p *resourceModel) getServiceConfig() *models.ServiceConfigReq { @@ -92,17 +101,18 @@ func (p *resourceModel) getServiceConfig() *models.ServiceConfigReq { } func (p *resourceModel) getTags() []*models.TagRequest { - if p.AwsOptions.Tags == nil { + if p.AwsOptions.Tags.IsNull() { return nil } - tags := make([]*models.TagRequest, len(*p.AwsOptions.Tags)) - for _, tag := range *p.AwsOptions.Tags { - if tag.Key.IsNull() || tag.Value.IsNull() { + tags := make([]*models.TagRequest, len(p.AwsOptions.Tags.Elements())) + for k, v := range p.AwsOptions.Tags.Elements() { + if v.IsNull() { continue } + value := v.String() tags = append(tags, &models.TagRequest{ - Key: tag.Key.ValueStringPointer(), - Value: tag.Value.ValueStringPointer(), + Key: &k, + Value: &value, }) } return tags diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go index a553ea4d..3194a8eb 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go @@ -119,6 +119,13 @@ func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, r plan.DatabaseCatalogID = types.StringValue(hive.Vw.DbcID) plan.Name = types.StringValue(hive.Vw.Name) plan.Status = types.StringValue(hive.Vw.Status) + plan.ImageVersion = types.StringValue(hive.Vw.CdhVersion) + plan.Compactor = types.BoolValue(hive.Vw.Compactor) + plan.JdbcUrl = types.StringValue(hive.Vw.Endpoints.HiveJdbc) + plan.KerberosJdbcUrl = types.StringValue(hive.Vw.Endpoints.HiveKerberosJdbc) + plan.HueUrl = types.StringValue(hive.Vw.Endpoints.Hue) + plan.JwtConnectionString = types.StringValue(hive.Vw.Endpoints.JwtConnectionString) + plan.JwtTokenGenUrl = types.StringValue(hive.Vw.Endpoints.JwtTokenGenURL) plan.LastUpdated = types.StringValue(time.Now().Format(time.RFC850)) diags = resp.State.Set(ctx, plan) resp.Diagnostics.Append(diags...) diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go index cf9f4ced..73971cc7 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go @@ -80,6 +80,20 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { cluster_id = %[1]q database_catalog_id = %[2]q name = %[3]q + node_count = 2 + platform_jwt_auth = true + enable_sso = true + autoscaling = { + min_clusters = 2 + max_clusters = 5 // Min: 2, Max: 20. Both values must be a multiple of 2. + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + hive_scale_wait_time_seconds = 230 + } + aws_options = { + availability_zone = "us-west-2a" + ebs_llap_spill_gb = 300 + } } `, params.ClusterID, params.DatabaseCatalogID, params.Name) } diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go index b107ffb0..d2bcd38f 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go @@ -24,6 +24,7 @@ import ( "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" "github.com/hashicorp/terraform-plugin-framework/tfsdk" + "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hashicorp/terraform-plugin-go/tftypes" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" @@ -62,6 +63,122 @@ var testHiveSchema = schema.Schema{ Required: true, MarkdownDescription: "The name of the Hive Virtual Warehouse.", }, + "image_version": schema.StringAttribute{ + Optional: true, + Computed: true, + MarkdownDescription: "The version of the Hive Virtual Warehouse image.", + }, + "node_count": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Nodes per compute cluster. If specified, forces ‘template’ to be ‘custom’.", + }, + "platform_jwt_auth": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Value of ‘true’ automatically configures the Virtual Warehouse to support JWTs issued by the CDP JWT token provider. Value of ‘false’ does not enable JWT auth on the Virtual Warehouse. If this field is not specified, it defaults to ‘false’.", + }, + "ldap_groups": schema.ListAttribute{ + Optional: true, + ElementType: types.StringType, + MarkdownDescription: "LDAP group names to be enabled for auth.", + }, + "enable_sso": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Enable SSO for the Virtual Warehouse. If this field is not specified, it defaults to ‘false’.", + }, + "compactor": schema.BoolAttribute{ + Computed: true, + MarkdownDescription: "Boolean value that describes if the Hive Virtual Warehouse is a compactor.", + }, + "jdbc_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "JDBC URL for the Hive Virtual Warehouse.", + }, + "kerberos_jdbc_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "Kerberos JDBC URL for the Hive Virtual Warehouse.", + }, + "hue_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "Hue URL for the Hive Virtual Warehouse.", + }, + "jwt_connection_string": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "Generic semi-colon delimited list of key-value pairs that contain all necessary information for clients to construct a connection to this Virtual Warehouse using JWTs as the authentication method.", + }, + "jwt_token_gen_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "URL to generate JWT tokens for the Virtual Warehouse by the CDP JWT token provider. Available if platform JWT authentication is enabled.", + }, + "autoscaling": schema.SingleNestedAttribute{ + MarkdownDescription: "Autoscaling related configuration options that could specify various values that will be used during CDW resource creation.", + Optional: true, + Attributes: map[string]schema.Attribute{ + "min_clusters": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Minimum number of available compute groups.", + }, + "max_clusters": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Maximum number of available compute groups.", + }, + "disable_auto_suspend": schema.BoolAttribute{ + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Boolean value that specifies if auto-suspend should be disabled.", + }, + "auto_suspend_timeout_seconds": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "The time in seconds after which the compute group should be suspended.", + }, + "hive_scale_wait_time_seconds": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", + }, + "hive_desired_free_capacity": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set Desired free capacity. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", + }, + }, + }, + "aws_options": schema.SingleNestedAttribute{ + MarkdownDescription: "AWS related configuration options that could specify various values that will be used during CDW resource creation.", + Optional: true, + Attributes: map[string]schema.Attribute{ + "availability_zone": schema.StringAttribute{ + Optional: true, + Computed: true, + MarkdownDescription: "This feature works only for AWS cluster type. An availability zone to host compute instances. If not specified, defaults to a randomly selected availability zone inferred from available subnets.", + }, + "ebs_llap_spill_gb": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "This feature works only for AWS cluster type. The size of the EBS volume in GB to be used for LLAP spill storage. If not specified, defaults to no extra spill disk.", + }, + "tags": schema.MapAttribute{ + Optional: true, + ElementType: types.StringType, + MarkdownDescription: "This feature works only for AWS cluster type. Tags to be applied to the underlying compute nodes.", + }, + }, + }, + "query_isolation_options": schema.SingleNestedAttribute{ + MarkdownDescription: "Query isolation related configuration options.", + Optional: true, + Attributes: map[string]schema.Attribute{ + "max_queries": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", + }, + "max_nodes_per_query": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", + }, + }, + }, "last_updated": schema.StringAttribute{ Computed: true, MarkdownDescription: "Timestamp of the last Terraform update of the order.", @@ -104,7 +221,7 @@ type MockTransport struct { runtime.ClientTransport } -func NewDwApi(client *mocks.MockDwClientService) *hiveResource { +func newDwApi(client *mocks.MockDwClientService) *hiveResource { return &hiveResource{ client: &cdp.Client{ Dw: &dwclient.Dw{ @@ -121,8 +238,44 @@ func createRawHiveResource() tftypes.Value { "cluster_id": tftypes.String, "database_catalog_id": tftypes.String, "name": tftypes.String, - "last_updated": tftypes.String, - "status": tftypes.String, + "image_version": tftypes.String, + "node_count": tftypes.Number, + "platform_jwt_auth": tftypes.Bool, + "ldap_groups": tftypes.List{ + ElementType: tftypes.String, + }, + "enable_sso": tftypes.Bool, + "compactor": tftypes.Bool, + "jdbc_url": tftypes.String, + "kerberos_jdbc_url": tftypes.String, + "hue_url": tftypes.String, + "jwt_connection_string": tftypes.String, + "jwt_token_gen_url": tftypes.String, + "autoscaling": tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "min_clusters": tftypes.Number, + "max_clusters": tftypes.Number, + "disable_auto_suspend": tftypes.Bool, + "auto_suspend_timeout_seconds": tftypes.Number, + "hive_scale_wait_time_seconds": tftypes.Number, + "hive_desired_free_capacity": tftypes.Number, + }, + }, + "aws_options": tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "availability_zone": tftypes.String, + "ebs_llap_spill_gb": tftypes.Number, + "tags": tftypes.Map{ElementType: tftypes.String}, + }, + }, + "query_isolation_options": tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "max_queries": tftypes.Number, + "max_nodes_per_query": tftypes.Number, + }, + }, + "last_updated": tftypes.String, + "status": tftypes.String, "polling_options": tftypes.Object{ AttributeTypes: map[string]tftypes.Type{ "async": tftypes.Bool, @@ -132,12 +285,63 @@ func createRawHiveResource() tftypes.Value { }, }}, map[string]tftypes.Value{ - "id": tftypes.NewValue(tftypes.String, ""), - "cluster_id": tftypes.NewValue(tftypes.String, "cluster-id"), - "database_catalog_id": tftypes.NewValue(tftypes.String, "database-catalog-id"), - "name": tftypes.NewValue(tftypes.String, ""), - "last_updated": tftypes.NewValue(tftypes.String, ""), - "status": tftypes.NewValue(tftypes.String, "Running"), + "id": tftypes.NewValue(tftypes.String, ""), + "cluster_id": tftypes.NewValue(tftypes.String, "cluster-id"), + "database_catalog_id": tftypes.NewValue(tftypes.String, "database-catalog-id"), + "name": tftypes.NewValue(tftypes.String, ""), + "image_version": tftypes.NewValue(tftypes.String, ""), + "node_count": tftypes.NewValue(tftypes.Number, 10), + "platform_jwt_auth": tftypes.NewValue(tftypes.Bool, true), + "ldap_groups": tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{}), + "enable_sso": tftypes.NewValue(tftypes.Bool, true), + "compactor": tftypes.NewValue(tftypes.Bool, false), + "jdbc_url": tftypes.NewValue(tftypes.String, ""), + "kerberos_jdbc_url": tftypes.NewValue(tftypes.String, ""), + "hue_url": tftypes.NewValue(tftypes.String, ""), + "jwt_connection_string": tftypes.NewValue(tftypes.String, ""), + "jwt_token_gen_url": tftypes.NewValue(tftypes.String, ""), + "autoscaling": tftypes.NewValue( + tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "min_clusters": tftypes.Number, + "max_clusters": tftypes.Number, + "disable_auto_suspend": tftypes.Bool, + "auto_suspend_timeout_seconds": tftypes.Number, + "hive_scale_wait_time_seconds": tftypes.Number, + "hive_desired_free_capacity": tftypes.Number, + }}, map[string]tftypes.Value{ + "min_clusters": tftypes.NewValue(tftypes.Number, 1), + "max_clusters": tftypes.NewValue(tftypes.Number, 10), + "disable_auto_suspend": tftypes.NewValue(tftypes.Bool, false), + "auto_suspend_timeout_seconds": tftypes.NewValue(tftypes.Number, 60), + "hive_scale_wait_time_seconds": tftypes.NewValue(tftypes.Number, 60), + "hive_desired_free_capacity": tftypes.NewValue(tftypes.Number, 10), + }), + "aws_options": tftypes.NewValue( + tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "availability_zone": tftypes.String, + "ebs_llap_spill_gb": tftypes.Number, + "tags": tftypes.Map{ElementType: tftypes.String}, + }}, map[string]tftypes.Value{ + "availability_zone": tftypes.NewValue(tftypes.String, "us-west-2a"), + "ebs_llap_spill_gb": tftypes.NewValue(tftypes.Number, 300), + "tags": tftypes.NewValue(tftypes.Map{ElementType: tftypes.String}, map[string]tftypes.Value{ + "key1": tftypes.NewValue(tftypes.String, "value1"), + "owner": tftypes.NewValue(tftypes.String, "dw-terraform@cloudera.com"), + }), + }), + "query_isolation_options": tftypes.NewValue( + tftypes.Object{ + AttributeTypes: map[string]tftypes.Type{ + "max_queries": tftypes.Number, + "max_nodes_per_query": tftypes.Number, + }}, map[string]tftypes.Value{ + "max_queries": tftypes.NewValue(tftypes.Number, 10), + "max_nodes_per_query": tftypes.NewValue(tftypes.Number, 10), + }), + "last_updated": tftypes.NewValue(tftypes.String, ""), + "status": tftypes.NewValue(tftypes.String, "Running"), "polling_options": tftypes.NewValue( tftypes.Object{ AttributeTypes: map[string]tftypes.Type{ @@ -169,7 +373,7 @@ func (suite *HiveTestSuite) SetupTest() { } func (suite *HiveTestSuite) TestHiveMetadata() { - dwApi := NewDwApi(new(mocks.MockDwClientService)) + dwApi := newDwApi(new(mocks.MockDwClientService)) resp := resource.MetadataResponse{} // Function under test @@ -182,7 +386,7 @@ func (suite *HiveTestSuite) TestHiveMetadata() { } func (suite *HiveTestSuite) TestHiveSchema() { - dwApi := NewDwApi(new(mocks.MockDwClientService)) + dwApi := newDwApi(new(mocks.MockDwClientService)) resp := resource.SchemaResponse{} // Function under test @@ -199,16 +403,24 @@ func (suite *HiveTestSuite) TestHiveCreate_Success() { expectedDescribeResponse := &operations.DescribeVwOK{ Payload: &models.DescribeVwResponse{ Vw: &models.VwSummary{ - ID: "test-id", - DbcID: "database-catalog-id", - Name: "test-name", - VwType: models.VwTypeHive, + ID: "test-id", + DbcID: "database-catalog-id", + Name: "test-name", + VwType: models.VwTypeHive, + Compactor: true, + Endpoints: &models.VwSummaryEndpoints{ + HiveJdbc: "jdbc://hive", + HiveKerberosJdbc: "jdbc://hive", + Hue: "https://hue", + JwtConnectionString: "connection-string", + JwtTokenGenURL: "https://jwt", + }, }}} client := new(mocks.MockDwClientService) client.On("CreateVw", mock.Anything).Return(suite.expectedCreateResponse, nil) client.On("DescribeVw", mock.Anything).Return(expectedDescribeResponse, nil) - dwApi := NewDwApi(client) + dwApi := newDwApi(client) req := resource.CreateRequest{ Plan: tfsdk.Plan{ @@ -239,7 +451,7 @@ func (suite *HiveTestSuite) TestHiveCreate_CreationError() { client := new(mocks.MockDwClientService) client.On("CreateVw", mock.Anything).Return(&operations.CreateVwOK{}, fmt.Errorf("create failed")) client.On("DescribeVw", mock.Anything).Return(&operations.DescribeVwOK{}, nil) - dwApi := NewDwApi(client) + dwApi := newDwApi(client) req := resource.CreateRequest{ Plan: tfsdk.Plan{ @@ -268,7 +480,7 @@ func (suite *HiveTestSuite) TestHiveCreate_DescribeError() { client := new(mocks.MockDwClientService) client.On("CreateVw", mock.Anything).Return(suite.expectedCreateResponse, nil) client.On("DescribeVw", mock.Anything).Return(&operations.DescribeVwOK{}, fmt.Errorf("describe failed")) - dwApi := NewDwApi(client) + dwApi := newDwApi(client) req := resource.CreateRequest{ Plan: tfsdk.Plan{ @@ -296,7 +508,7 @@ func (suite *HiveTestSuite) TestHiveDeletion_Success() { ctx := context.TODO() client := new(mocks.MockDwClientService) client.On("DeleteVw", mock.Anything).Return(&operations.DeleteVwOK{}, nil) - dwApi := NewDwApi(client) + dwApi := newDwApi(client) req := resource.DeleteRequest{ State: tfsdk.State{ @@ -316,7 +528,7 @@ func (suite *HiveTestSuite) TestHiveDeletion_ReturnsError() { ctx := context.TODO() client := new(mocks.MockDwClientService) client.On("DeleteVw", mock.Anything).Return(&operations.DeleteVwOK{}, fmt.Errorf("delete failed")) - dwApi := NewDwApi(client) + dwApi := newDwApi(client) req := resource.DeleteRequest{ State: tfsdk.State{ @@ -345,7 +557,7 @@ func (suite *HiveTestSuite) TestStateRefresh_Success() { }, }, nil) - dwApi := NewDwApi(client) + dwApi := newDwApi(client) clusterID := "cluster-id" vwID := "hive-id" @@ -364,7 +576,7 @@ func (suite *HiveTestSuite) TestStateRefresh_FailureThresholdReached() { client := new(mocks.MockDwClientService) client.On("DescribeVw", mock.Anything).Return( &operations.DescribeVwOK{}, fmt.Errorf("unknown error")) - dwApi := NewDwApi(client) + dwApi := newDwApi(client) clusterID := "cluster-id" vwID := "hive-id" diff --git a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go index a49ac436..bec00913 100644 --- a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go @@ -48,7 +48,8 @@ var hiveSchema = schema.Schema{ MarkdownDescription: "The name of the Hive Virtual Warehouse.", }, "image_version": schema.StringAttribute{ - Required: true, + Optional: true, + Computed: true, MarkdownDescription: "The version of the Hive Virtual Warehouse image.", }, "node_count": schema.Int64Attribute{ @@ -72,6 +73,30 @@ var hiveSchema = schema.Schema{ Default: booldefault.StaticBool(false), MarkdownDescription: "Enable SSO for the Virtual Warehouse. If this field is not specified, it defaults to ‘false’.", }, + "compactor": schema.BoolAttribute{ + Computed: true, + MarkdownDescription: "Boolean value that describes if the Hive Virtual Warehouse is a compactor.", + }, + "jdbc_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "JDBC URL for the Hive Virtual Warehouse.", + }, + "kerberos_jdbc_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "Kerberos JDBC URL for the Hive Virtual Warehouse.", + }, + "hue_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "Hue URL for the Hive Virtual Warehouse.", + }, + "jwt_connection_string": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "Generic semi-colon delimited list of key-value pairs that contain all necessary information for clients to construct a connection to this Virtual Warehouse using JWTs as the authentication method.", + }, + "jwt_token_gen_url": schema.StringAttribute{ + Computed: true, + MarkdownDescription: "URL to generate JWT tokens for the Virtual Warehouse by the CDP JWT token provider. Available if platform JWT authentication is enabled.", + }, "autoscaling": schema.SingleNestedAttribute{ MarkdownDescription: "Autoscaling related configuration options that could specify various values that will be used during CDW resource creation.", Optional: true, @@ -85,14 +110,16 @@ var hiveSchema = schema.Schema{ MarkdownDescription: "Maximum number of available compute groups.", }, "disable_auto_suspend": schema.BoolAttribute{ - Required: true, + Optional: true, + Computed: true, + Default: booldefault.StaticBool(false), MarkdownDescription: "Boolean value that specifies if auto-suspend should be disabled.", }, "auto_suspend_timeout_seconds": schema.Int64Attribute{ - Required: true, + Optional: true, MarkdownDescription: "The time in seconds after which the compute group should be suspended.", }, - "hive_scale_wait_time_seconds": schema.Int64Attribute{ + "hive_scale_wait_time_seconds": schema.Int64Attribute{ // TODO add validation for the options set Optional: true, MarkdownDescription: "Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", }, @@ -115,7 +142,7 @@ var hiveSchema = schema.Schema{ Optional: true, MarkdownDescription: "This feature works only for AWS cluster type. The size of the EBS volume in GB to be used for LLAP spill storage. If not specified, defaults to no extra spill disk.", }, - "tags": schema.ListAttribute{ + "tags": schema.MapAttribute{ Optional: true, ElementType: types.StringType, MarkdownDescription: "This feature works only for AWS cluster type. Tags to be applied to the underlying compute nodes.", From 541194ee580c5fb929b69cc912ddf35d8f7f7ecc Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Wed, 4 Dec 2024 21:46:35 -0600 Subject: [PATCH 3/9] Add tag property The AWS tags should annotate the compute nodes for the customer. The creation parameters are passed in this change. --- resources/dw/virtualwarehouse/hive/model_hive_vw.go | 11 ++++++----- .../dw/virtualwarehouse/hive/resource_hive_vw.go | 1 + .../hive/resource_hive_vw_acc_test.go | 3 +++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/resources/dw/virtualwarehouse/hive/model_hive_vw.go b/resources/dw/virtualwarehouse/hive/model_hive_vw.go index 8fb770e4..060f2f7d 100644 --- a/resources/dw/virtualwarehouse/hive/model_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/model_hive_vw.go @@ -12,6 +12,7 @@ package hive import ( "github.com/hashicorp/terraform-plugin-framework/types" + "strings" "github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/dw/models" "github.com/cloudera/terraform-provider-cdp/utils" @@ -78,9 +79,9 @@ func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { QueryIsolationOptions: p.getQueryIsolationOptions(), Autoscaling: p.getAutoscaling(), AvailabilityZone: p.getAvailabilityZone(), - //Tags: p.getTags(), - Config: p.getServiceConfig(), - VwType: &vwType, + Tags: p.getTags(), + Config: p.getServiceConfig(), + VwType: &vwType, } } @@ -104,12 +105,12 @@ func (p *resourceModel) getTags() []*models.TagRequest { if p.AwsOptions.Tags.IsNull() { return nil } - tags := make([]*models.TagRequest, len(p.AwsOptions.Tags.Elements())) + var tags []*models.TagRequest for k, v := range p.AwsOptions.Tags.Elements() { if v.IsNull() { continue } - value := v.String() + value := strings.TrimPrefix(strings.TrimSuffix(v.String(), "\""), "\"") tags = append(tags, &models.TagRequest{ Key: &k, Value: &value, diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go index 3194a8eb..0c9cb1e7 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go @@ -115,6 +115,7 @@ func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, r } hive := describe.GetPayload() + // TODO move this to a converter or similar plan.ID = types.StringValue(hive.Vw.ID) plan.DatabaseCatalogID = types.StringValue(hive.Vw.DbcID) plan.Name = types.StringValue(hive.Vw.Name) diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go index 73971cc7..a562f8c8 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go @@ -93,6 +93,9 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { aws_options = { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 + tags = { + owner = "cdw-terraform@cloudera.com" + } } } `, params.ClusterID, params.DatabaseCatalogID, params.Name) From 7e36e6e647639c7c57d1362a54df65cfd9833f7d Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Mon, 9 Dec 2024 21:58:31 -0600 Subject: [PATCH 4/9] Add remaining options and tests Image version and acceptance tests. --- resources/dw/resource_dw_acc_test.go | 18 +++++++++++ .../dw/virtualwarehouse/hive/model_hive_vw.go | 31 ++++++++++++++----- .../virtualwarehouse/hive/resource_hive_vw.go | 18 +---------- .../hive/resource_hive_vw_acc_test.go | 3 +- .../virtualwarehouse/hive/schema_hive_vw.go | 2 +- 5 files changed, 46 insertions(+), 26 deletions(-) diff --git a/resources/dw/resource_dw_acc_test.go b/resources/dw/resource_dw_acc_test.go index c70e7c72..4300051a 100644 --- a/resources/dw/resource_dw_acc_test.go +++ b/resources/dw/resource_dw_acc_test.go @@ -283,6 +283,24 @@ func testAccHiveVirtualWarehouse(name string) string { cluster_id = cdp_dw_aws_cluster.test_data_warehouse_aws.cluster_id database_catalog_id = cdp_dw_database_catalog.test_catalog.id name = %[1]q + platform_jwt_auth = true + enable_sso = true + image_version = "2024.0.19.0-301" + node_count = 2 + autoscaling = { + min_clusters = 1 + max_clusters = 3 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + hive_scale_wait_time_seconds = 230 + } + aws_options = { + availability_zone = "us-west-2a" + ebs_llap_spill_gb = 300 + tags = { + owner = "cdw-terraform@cloudera.com" + } + } } `, name) } diff --git a/resources/dw/virtualwarehouse/hive/model_hive_vw.go b/resources/dw/virtualwarehouse/hive/model_hive_vw.go index 060f2f7d..318580b9 100644 --- a/resources/dw/virtualwarehouse/hive/model_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/model_hive_vw.go @@ -11,8 +11,10 @@ package hive import ( - "github.com/hashicorp/terraform-plugin-framework/types" "strings" + "time" + + "github.com/hashicorp/terraform-plugin-framework/types" "github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/dw/models" "github.com/cloudera/terraform-provider-cdp/utils" @@ -69,10 +71,10 @@ func (p *resourceModel) GetPollingOptions() *utils.PollingOptions { func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { vwType := models.VwType("hive") return &models.CreateVwRequest{ - ClusterID: p.ClusterID.ValueStringPointer(), - DbcID: p.DatabaseCatalogID.ValueStringPointer(), - EbsLLAPSpillGB: p.getEbsLLAPSpillGB(), - //ImageVersion: p.getImageVersion(), + ClusterID: p.ClusterID.ValueStringPointer(), + DbcID: p.DatabaseCatalogID.ValueStringPointer(), + EbsLLAPSpillGB: p.getEbsLLAPSpillGB(), + ImageVersion: p.getImageVersion(), Name: p.Name.ValueStringPointer(), NodeCount: utils.Int64To32(p.NodeCount), PlatformJwtAuth: p.PlatformJwtAuth.ValueBoolPointer(), @@ -86,10 +88,10 @@ func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { } func (p *resourceModel) getImageVersion() string { - if p.ImageVersion.IsNull() || p.ImageVersion.String() == "unknown" { + if p.ImageVersion.IsNull() || p.ImageVersion.String() == "" { return "" } - return p.ImageVersion.String() + return strings.TrimPrefix(strings.TrimSuffix(p.ImageVersion.String(), "\""), "\"") } func (p *resourceModel) getServiceConfig() *models.ServiceConfigReq { @@ -156,3 +158,18 @@ func (p *resourceModel) getAutoscaling() *models.AutoscalingOptionsCreateRequest HiveDesiredFreeCapacity: utils.Int64To32(p.Autoscaling.HiveDesiredFreeCapacity), } } + +func (p *resourceModel) setFromDescribeVwResponse(resp *models.DescribeVwResponse) { + p.ID = types.StringValue(resp.Vw.ID) + p.DatabaseCatalogID = types.StringValue(resp.Vw.DbcID) + p.Name = types.StringValue(resp.Vw.Name) + p.Status = types.StringValue(resp.Vw.Status) + p.ImageVersion = types.StringValue(resp.Vw.CdhVersion) + p.Compactor = types.BoolValue(resp.Vw.Compactor) + p.JdbcUrl = types.StringValue(resp.Vw.Endpoints.HiveJdbc) + p.KerberosJdbcUrl = types.StringValue(resp.Vw.Endpoints.HiveKerberosJdbc) + p.HueUrl = types.StringValue(resp.Vw.Endpoints.Hue) + p.JwtConnectionString = types.StringValue(resp.Vw.Endpoints.JwtConnectionString) + p.JwtTokenGenUrl = types.StringValue(resp.Vw.Endpoints.JwtTokenGenURL) + p.LastUpdated = types.StringValue(time.Now().Format(time.RFC850)) +} diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go index 0c9cb1e7..d7518ef6 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go @@ -18,7 +18,6 @@ import ( "github.com/hashicorp/terraform-plugin-framework/path" "github.com/hashicorp/terraform-plugin-framework/resource" - "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hashicorp/terraform-plugin-log/tflog" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/retry" @@ -59,7 +58,6 @@ func (r *hiveResource) Schema(_ context.Context, _ resource.SchemaRequest, resp } func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { - // Retrieve values from plan var plan resourceModel diags := req.Plan.Get(ctx, &plan) resp.Diagnostics.Append(diags...) @@ -67,11 +65,9 @@ func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, r return } - // Generate API request body from plan vw := operations.NewCreateVwParamsWithContext(ctx). WithInput(plan.convertToCreateVwRequest()) - // Create new virtual warehouse response, err := r.client.Dw.Operations.CreateVw(vw) if err != nil { resp.Diagnostics.AddError( @@ -115,19 +111,7 @@ func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, r } hive := describe.GetPayload() - // TODO move this to a converter or similar - plan.ID = types.StringValue(hive.Vw.ID) - plan.DatabaseCatalogID = types.StringValue(hive.Vw.DbcID) - plan.Name = types.StringValue(hive.Vw.Name) - plan.Status = types.StringValue(hive.Vw.Status) - plan.ImageVersion = types.StringValue(hive.Vw.CdhVersion) - plan.Compactor = types.BoolValue(hive.Vw.Compactor) - plan.JdbcUrl = types.StringValue(hive.Vw.Endpoints.HiveJdbc) - plan.KerberosJdbcUrl = types.StringValue(hive.Vw.Endpoints.HiveKerberosJdbc) - plan.HueUrl = types.StringValue(hive.Vw.Endpoints.Hue) - plan.JwtConnectionString = types.StringValue(hive.Vw.Endpoints.JwtConnectionString) - plan.JwtTokenGenUrl = types.StringValue(hive.Vw.Endpoints.JwtTokenGenURL) - plan.LastUpdated = types.StringValue(time.Now().Format(time.RFC850)) + plan.setFromDescribeVwResponse(hive) diags = resp.State.Set(ctx, plan) resp.Diagnostics.Append(diags...) } diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go index a562f8c8..68a05cbe 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go @@ -83,9 +83,10 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { node_count = 2 platform_jwt_auth = true enable_sso = true + image_version = "2024.0.19.0-301" autoscaling = { min_clusters = 2 - max_clusters = 5 // Min: 2, Max: 20. Both values must be a multiple of 2. + max_clusters = 5 disable_auto_suspend = false auto_suspend_timeout_seconds = 100 hive_scale_wait_time_seconds = 230 diff --git a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go index bec00913..059525bb 100644 --- a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go @@ -119,7 +119,7 @@ var hiveSchema = schema.Schema{ Optional: true, MarkdownDescription: "The time in seconds after which the compute group should be suspended.", }, - "hive_scale_wait_time_seconds": schema.Int64Attribute{ // TODO add validation for the options set + "hive_scale_wait_time_seconds": schema.Int64Attribute{ Optional: true, MarkdownDescription: "Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", }, From 87e11207213fc31e4deec629b7e330d8a9bf24a7 Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Mon, 9 Dec 2024 22:10:22 -0600 Subject: [PATCH 5/9] Render documentation with new properties --- docs/resources/dw_vw_hive.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/resources/dw_vw_hive.md b/docs/resources/dw_vw_hive.md index e673f2a8..9858b935 100644 --- a/docs/resources/dw_vw_hive.md +++ b/docs/resources/dw_vw_hive.md @@ -36,7 +36,6 @@ resource "cdp_dw_vw_hive" "example" { - `cluster_id` (String) The id of the CDW Cluster which the Hive Virtual Warehouse is attached to. - `database_catalog_id` (String) The id of the Database Catalog which the Hive Virtual Warehouse is attached to. -- `image_version` (String) The version of the Hive Virtual Warehouse image. - `name` (String) The name of the Hive Virtual Warehouse. ### Optional @@ -44,6 +43,7 @@ resource "cdp_dw_vw_hive" "example" { - `autoscaling` (Attributes) Autoscaling related configuration options that could specify various values that will be used during CDW resource creation. (see [below for nested schema](#nestedatt--autoscaling)) - `aws_options` (Attributes) AWS related configuration options that could specify various values that will be used during CDW resource creation. (see [below for nested schema](#nestedatt--aws_options)) - `enable_sso` (Boolean) Enable SSO for the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. +- `image_version` (String) The version of the Hive Virtual Warehouse image. - `ldap_groups` (List of String) LDAP group names to be enabled for auth. - `node_count` (Number) Nodes per compute cluster. If specified, forces ‘template’ to be ‘custom’. - `platform_jwt_auth` (Boolean) Value of ‘true’ automatically configures the Virtual Warehouse to support JWTs issued by the CDP JWT token provider. Value of ‘false’ does not enable JWT auth on the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. @@ -52,7 +52,13 @@ resource "cdp_dw_vw_hive" "example" { ### Read-Only +- `compactor` (Boolean) Boolean value that describes if the Hive Virtual Warehouse is a compactor. +- `hue_url` (String) Hue URL for the Hive Virtual Warehouse. - `id` (String) The ID of this resource. +- `jdbc_url` (String) JDBC URL for the Hive Virtual Warehouse. +- `jwt_connection_string` (String) Generic semi-colon delimited list of key-value pairs that contain all necessary information for clients to construct a connection to this Virtual Warehouse using JWTs as the authentication method. +- `jwt_token_gen_url` (String) URL to generate JWT tokens for the Virtual Warehouse by the CDP JWT token provider. Available if platform JWT authentication is enabled. +- `kerberos_jdbc_url` (String) Kerberos JDBC URL for the Hive Virtual Warehouse. - `last_updated` (String) Timestamp of the last Terraform update of the order. - `status` (String) The status of the database catalog. @@ -61,13 +67,13 @@ resource "cdp_dw_vw_hive" "example" { Required: -- `auto_suspend_timeout_seconds` (Number) The time in seconds after which the compute group should be suspended. -- `disable_auto_suspend` (Boolean) Boolean value that specifies if auto-suspend should be disabled. - `max_clusters` (Number) Maximum number of available compute groups. - `min_clusters` (Number) Minimum number of available compute groups. Optional: +- `auto_suspend_timeout_seconds` (Number) The time in seconds after which the compute group should be suspended. +- `disable_auto_suspend` (Boolean) Boolean value that specifies if auto-suspend should be disabled. - `hive_desired_free_capacity` (Number) Set Desired free capacity. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided. - `hive_scale_wait_time_seconds` (Number) Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided. @@ -79,7 +85,7 @@ Optional: - `availability_zone` (String) This feature works only for AWS cluster type. An availability zone to host compute instances. If not specified, defaults to a randomly selected availability zone inferred from available subnets. - `ebs_llap_spill_gb` (Number) This feature works only for AWS cluster type. The size of the EBS volume in GB to be used for LLAP spill storage. If not specified, defaults to no extra spill disk. -- `tags` (List of String) This feature works only for AWS cluster type. Tags to be applied to the underlying compute nodes. +- `tags` (Map of String) This feature works only for AWS cluster type. Tags to be applied to the underlying compute nodes. @@ -99,5 +105,3 @@ Optional: - `max_nodes_per_query` (Number) Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. - `max_queries` (Number) Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. - - From b42887f2c25a05a9dda71687102fa32bfd36043a Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Tue, 10 Dec 2024 22:28:57 -0600 Subject: [PATCH 6/9] Adding unit tests and example resource with all properties --- docs/resources/dw_vw_hive.md | 29 +++++- examples/resources/cdp_dw_vw_hive/resource.tf | 29 +++++- resources/dw/resource_dw_acc_test.go | 10 +- .../dw/virtualwarehouse/hive/model_hive_vw.go | 2 +- .../virtualwarehouse/hive/resource_hive_vw.go | 4 +- .../hive/resource_hive_vw_acc_test.go | 11 ++- .../hive/resource_hive_vw_test.go | 94 +++++++++++++++++++ 7 files changed, 164 insertions(+), 15 deletions(-) diff --git a/docs/resources/dw_vw_hive.md b/docs/resources/dw_vw_hive.md index 9858b935..9c2978a1 100644 --- a/docs/resources/dw_vw_hive.md +++ b/docs/resources/dw_vw_hive.md @@ -23,9 +23,32 @@ A Hive Virtual Warehouse is service which is able to run big SQL queries. # permissions and limitations governing your use of the file. resource "cdp_dw_vw_hive" "example" { - cluster_id = var.cluster_id - database_catalog_id = var.database_catalog_id - name = var.name + cluster_id = "env-id" + database_catalog_id = "warehouse-id" + name = "default-catalog" + node_count = 2 + platform_jwt_auth = true + enable_sso = true + image_version = "2024.0.19.0-301" + autoscaling = { + min_clusters = 1 + max_clusters = 3 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + hive_scale_wait_time_seconds = 230 + hive_desired_free_capacity = 1 + } + aws_options = { + availability_zone = "us-west-2a" + ebs_llap_spill_gb = 300 + tags = { + "key1" = "value1" + } + } + query_isolation_options = { + max_queries = 100 + max_nodes_per_query = 10 + } } ``` diff --git a/examples/resources/cdp_dw_vw_hive/resource.tf b/examples/resources/cdp_dw_vw_hive/resource.tf index c4094659..04b75c76 100644 --- a/examples/resources/cdp_dw_vw_hive/resource.tf +++ b/examples/resources/cdp_dw_vw_hive/resource.tf @@ -9,7 +9,30 @@ # permissions and limitations governing your use of the file. resource "cdp_dw_vw_hive" "example" { - cluster_id = var.cluster_id - database_catalog_id = var.database_catalog_id - name = var.name + cluster_id = "env-id" + database_catalog_id = "warehouse-id" + name = "default-catalog" + node_count = 2 + platform_jwt_auth = true + enable_sso = true + image_version = "2024.0.19.0-301" + autoscaling = { + min_clusters = 1 + max_clusters = 3 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + hive_scale_wait_time_seconds = 230 + hive_desired_free_capacity = 1 + } + aws_options = { + availability_zone = "us-west-2a" + ebs_llap_spill_gb = 300 + tags = { + "key1" = "value1" + } + } + query_isolation_options = { + max_queries = 100 + max_nodes_per_query = 10 + } } diff --git a/resources/dw/resource_dw_acc_test.go b/resources/dw/resource_dw_acc_test.go index 4300051a..dc2a073b 100644 --- a/resources/dw/resource_dw_acc_test.go +++ b/resources/dw/resource_dw_acc_test.go @@ -284,11 +284,11 @@ func testAccHiveVirtualWarehouse(name string) string { database_catalog_id = cdp_dw_database_catalog.test_catalog.id name = %[1]q platform_jwt_auth = true - enable_sso = true - image_version = "2024.0.19.0-301" + enable_sso = true + image_version = "2024.0.19.0-301" node_count = 2 - autoscaling = { - min_clusters = 1 + autoscaling = { + min_clusters = 1 max_clusters = 3 disable_auto_suspend = false auto_suspend_timeout_seconds = 100 @@ -298,7 +298,7 @@ func testAccHiveVirtualWarehouse(name string) string { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 tags = { - owner = "cdw-terraform@cloudera.com" + owner = "cdw-terraform" } } } diff --git a/resources/dw/virtualwarehouse/hive/model_hive_vw.go b/resources/dw/virtualwarehouse/hive/model_hive_vw.go index 318580b9..0bfc2137 100644 --- a/resources/dw/virtualwarehouse/hive/model_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/model_hive_vw.go @@ -88,7 +88,7 @@ func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { } func (p *resourceModel) getImageVersion() string { - if p.ImageVersion.IsNull() || p.ImageVersion.String() == "" { + if p.ImageVersion.IsNull() || p.ImageVersion.IsUnknown() || p.ImageVersion.String() == "" { return "" } return strings.TrimPrefix(strings.TrimSuffix(p.ImageVersion.String(), "\""), "\"") diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go index d7518ef6..69aeb03f 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go @@ -81,7 +81,7 @@ func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, r clusterID := plan.ClusterID.ValueStringPointer() vwID := &payload.VwID - if opts := plan.PollingOptions; !(opts != nil && opts.Async.ValueBool()) { + if opts := plan.PollingOptions; opts == nil || !opts.Async.ValueBool() { callFailedCount := 0 stateConf := &retry.StateChangeConf{ Pending: []string{"Accepted", "Creating", "Created", "Starting"}, @@ -151,7 +151,7 @@ func (r *hiveResource) Delete(ctx context.Context, req resource.DeleteRequest, r return } - if opts := state.PollingOptions; !(opts != nil && opts.Async.ValueBool()) { + if opts := state.PollingOptions; opts == nil || !opts.Async.ValueBool() { callFailedCount := 0 stateConf := &retry.StateChangeConf{ Pending: []string{"Deleting", "Running", "Stopping", "Stopped", "Creating", "Created", "Starting", "Updating"}, diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go index 68a05cbe..0a4fc976 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go @@ -67,6 +67,12 @@ func TestAccHive_basic(t *testing.T) { resource.TestCheckResourceAttr("cdp_dw_vw_hive.test_hive", "name", params.Name), resource.TestCheckResourceAttr("cdp_dw_vw_hive.test_hive", "cluster_id", params.ClusterID), resource.TestCheckResourceAttr("cdp_dw_vw_hive.test_hive", "database_catalog_id", params.DatabaseCatalogID), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "compactor"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jdbc_url"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "kerberos_jdbc_url"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "hue_url"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jwt_connection_string"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jwt_token_gen_url"), ), }, // Delete testing automatically occurs in TestCase @@ -83,7 +89,6 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { node_count = 2 platform_jwt_auth = true enable_sso = true - image_version = "2024.0.19.0-301" autoscaling = { min_clusters = 2 max_clusters = 5 @@ -98,6 +103,10 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { owner = "cdw-terraform@cloudera.com" } } + query_isolation_options = { + max_queries = 100 + max_nodes_per_query = 10 + } } `, params.ClusterID, params.DatabaseCatalogID, params.Name) } diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go index d2bcd38f..c8e3c617 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go @@ -16,6 +16,7 @@ import ( "testing" "github.com/go-openapi/runtime" + "github.com/hashicorp/terraform-plugin-framework/attr" "github.com/hashicorp/terraform-plugin-framework/resource" "github.com/hashicorp/terraform-plugin-framework/resource/schema" "github.com/hashicorp/terraform-plugin-framework/resource/schema/booldefault" @@ -591,3 +592,96 @@ func (suite *HiveTestSuite) TestStateRefresh_FailureThresholdReached() { } suite.Error(err, "unknown error") } + +func (suite *HiveTestSuite) TestConvertToCreateVwRequest_All() { + plan := resourceModel{ + ClusterID: types.StringValue("cluster-id"), + DatabaseCatalogID: types.StringValue("database-catalog-id"), + Name: types.StringValue("test-name"), + ImageVersion: types.StringValue("2024.0.19.0-301"), + NodeCount: types.Int64Value(10), + PlatformJwtAuth: types.BoolValue(true), + LdapGroups: types.ListValueMust(types.StringType, []attr.Value{types.StringValue("ldap-group")}), + EnableSSO: types.BoolValue(true), + Autoscaling: &autoscaling{ + MinClusters: types.Int64Value(1), + MaxClusters: types.Int64Value(10), + DisableAutoSuspend: types.BoolValue(false), + AutoSuspendTimeoutSeconds: types.Int64Value(60), + HiveScaleWaitTimeSeconds: types.Int64Value(60), + HiveDesiredFreeCapacity: types.Int64Value(2), + }, + AwsOptions: &awsOptions{ + AvailabilityZone: types.StringValue("us-west-2a"), + EbsLLAPSpillGb: types.Int64Value(300), + Tags: types.MapValueMust(types.StringType, map[string]attr.Value{"key1": types.StringValue("value1")}), + }, + QueryIsolationOptions: &queryIsolationOptions{ + MaxQueries: types.Int64Value(5), + MaxNodesPerQuery: types.Int64Value(2), + }, + } + + req := plan.convertToCreateVwRequest() + suite.Equal("cluster-id", *req.ClusterID) + suite.Equal("database-catalog-id", *req.DbcID) + suite.Equal("test-name", *req.Name) + suite.Equal("2024.0.19.0-301", req.ImageVersion) + suite.Equal(int32(10), req.NodeCount) + suite.Equal(true, *req.PlatformJwtAuth) + suite.Equal([]string{"ldap-group"}, req.Config.LdapGroups) + suite.Equal(true, req.Config.EnableSSO) + suite.Equal(int32(1), *req.Autoscaling.MinClusters) + suite.Equal(int32(10), *req.Autoscaling.MaxClusters) + suite.Equal(false, req.Autoscaling.DisableAutoSuspend) + suite.Equal(int32(60), req.Autoscaling.AutoSuspendTimeoutSeconds) + suite.Equal(int32(60), req.Autoscaling.HiveScaleWaitTimeSeconds) + suite.Equal(int32(2), req.Autoscaling.HiveDesiredFreeCapacity) + suite.Equal("us-west-2a", req.AvailabilityZone) + suite.Equal(int32(300), req.EbsLLAPSpillGB) + suite.Equal("key1", *req.Tags[0].Key) + suite.Equal("value1", *req.Tags[0].Value) + suite.Equal(int32(5), req.QueryIsolationOptions.MaxQueries) + suite.Equal(int32(2), req.QueryIsolationOptions.MaxNodesPerQuery) +} + +func (suite *HiveTestSuite) TestConvertToCreateVwRequest_MissingImageVersion() { + plan := resourceModel{ + ImageVersion: types.StringUnknown(), + AwsOptions: &awsOptions{}, + Autoscaling: &autoscaling{}, + } + + req := plan.convertToCreateVwRequest() + suite.Equal("", req.ImageVersion) +} + +func (suite *HiveTestSuite) TestSetFromDescribeVwResponse() { + plan := resourceModel{} + resp := &models.DescribeVwResponse{ + Vw: &models.VwSummary{ + ID: "test-id", + DbcID: "database-catalog-id", + Name: "test-name", + Status: "Running", + Endpoints: &models.VwSummaryEndpoints{ + HiveJdbc: "jdbc://hive", + HiveKerberosJdbc: "jdbc://hive", + Hue: "https://hue", + JwtConnectionString: "connection-string", + JwtTokenGenURL: "https://jwt", + }, + }, + } + + plan.setFromDescribeVwResponse(resp) + suite.Equal("test-id", plan.ID.ValueString()) + suite.Equal("database-catalog-id", plan.DatabaseCatalogID.ValueString()) + suite.Equal("test-name", plan.Name.ValueString()) + suite.Equal("Running", plan.Status.ValueString()) + suite.Equal("jdbc://hive", plan.JdbcUrl.ValueString()) + suite.Equal("jdbc://hive", plan.KerberosJdbcUrl.ValueString()) + suite.Equal("https://hue", plan.HueUrl.ValueString()) + suite.Equal("connection-string", plan.JwtConnectionString.ValueString()) + suite.Equal("https://jwt", plan.JwtTokenGenUrl.ValueString()) +} From 27c1b72edddc959e732d0d1efe8c0c69e8af590c Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Sat, 14 Dec 2024 23:17:33 -0600 Subject: [PATCH 7/9] Refactor properties based on feedback Move the properties high level, rename some of them to a more meaningful name. --- .../dw/virtualwarehouse/hive/model_hive_vw.go | 86 +++---- .../hive/resource_hive_vw_acc_test.go | 22 +- .../hive/resource_hive_vw_test.go | 221 +++++++----------- .../virtualwarehouse/hive/schema_hive_vw.go | 80 +++---- 4 files changed, 167 insertions(+), 242 deletions(-) diff --git a/resources/dw/virtualwarehouse/hive/model_hive_vw.go b/resources/dw/virtualwarehouse/hive/model_hive_vw.go index 0bfc2137..85b98463 100644 --- a/resources/dw/virtualwarehouse/hive/model_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/model_hive_vw.go @@ -20,48 +20,40 @@ import ( "github.com/cloudera/terraform-provider-cdp/utils" ) -type autoscaling struct { - MinClusters types.Int64 `tfsdk:"min_clusters"` - MaxClusters types.Int64 `tfsdk:"max_clusters"` - DisableAutoSuspend types.Bool `tfsdk:"disable_auto_suspend"` - AutoSuspendTimeoutSeconds types.Int64 `tfsdk:"auto_suspend_timeout_seconds"` - HiveScaleWaitTimeSeconds types.Int64 `tfsdk:"hive_scale_wait_time_seconds"` - HiveDesiredFreeCapacity types.Int64 `tfsdk:"hive_desired_free_capacity"` -} - type awsOptions struct { AvailabilityZone types.String `tfsdk:"availability_zone"` EbsLLAPSpillGb types.Int64 `tfsdk:"ebs_llap_spill_gb"` Tags types.Map `tfsdk:"tags"` } -type queryIsolationOptions struct { - MaxQueries types.Int64 `tfsdk:"max_queries"` - MaxNodesPerQuery types.Int64 `tfsdk:"max_nodes_per_query"` -} - type resourceModel struct { - ID types.String `tfsdk:"id"` - ClusterID types.String `tfsdk:"cluster_id"` - DatabaseCatalogID types.String `tfsdk:"database_catalog_id"` - Name types.String `tfsdk:"name"` - ImageVersion types.String `tfsdk:"image_version"` - NodeCount types.Int64 `tfsdk:"node_count"` - PlatformJwtAuth types.Bool `tfsdk:"platform_jwt_auth"` - LdapGroups types.List `tfsdk:"ldap_groups"` - EnableSSO types.Bool `tfsdk:"enable_sso"` - Compactor types.Bool `tfsdk:"compactor"` - JdbcUrl types.String `tfsdk:"jdbc_url"` - KerberosJdbcUrl types.String `tfsdk:"kerberos_jdbc_url"` - HueUrl types.String `tfsdk:"hue_url"` - JwtConnectionString types.String `tfsdk:"jwt_connection_string"` - JwtTokenGenUrl types.String `tfsdk:"jwt_token_gen_url"` - Autoscaling *autoscaling `tfsdk:"autoscaling"` - AwsOptions *awsOptions `tfsdk:"aws_options"` - QueryIsolationOptions *queryIsolationOptions `tfsdk:"query_isolation_options"` - LastUpdated types.String `tfsdk:"last_updated"` - Status types.String `tfsdk:"status"` - PollingOptions *utils.PollingOptions `tfsdk:"polling_options"` + ID types.String `tfsdk:"id"` + ClusterID types.String `tfsdk:"cluster_id"` + DatabaseCatalogID types.String `tfsdk:"database_catalog_id"` + Name types.String `tfsdk:"name"` + ImageVersion types.String `tfsdk:"image_version"` + GroupSize types.Int64 `tfsdk:"group_size"` + PlatformJwtAuth types.Bool `tfsdk:"platform_jwt_auth"` + LdapGroups types.List `tfsdk:"ldap_groups"` + EnableSSO types.Bool `tfsdk:"enable_sso"` + Compactor types.Bool `tfsdk:"compactor"` + JdbcUrl types.String `tfsdk:"jdbc_url"` + KerberosJdbcUrl types.String `tfsdk:"kerberos_jdbc_url"` + HueUrl types.String `tfsdk:"hue_url"` + JwtConnectionString types.String `tfsdk:"jwt_connection_string"` + JwtTokenGenUrl types.String `tfsdk:"jwt_token_gen_url"` + MinGroupCount types.Int64 `tfsdk:"min_group_count"` + MaxGroupCount types.Int64 `tfsdk:"max_group_count"` + DisableAutoSuspend types.Bool `tfsdk:"disable_auto_suspend"` + AutoSuspendTimeoutSeconds types.Int64 `tfsdk:"auto_suspend_timeout_seconds"` + ScaleWaitTimeSeconds types.Int64 `tfsdk:"scale_wait_time_seconds"` + Headroom types.Int64 `tfsdk:"headroom"` + MaxConcurrentIsolatedQueries types.Int64 `tfsdk:"max_concurrent_isolated_queries"` + MaxNodesPerIsolatedQuery types.Int64 `tfsdk:"max_nodes_per_isolated_query"` + AwsOptions *awsOptions `tfsdk:"aws_options"` + LastUpdated types.String `tfsdk:"last_updated"` + Status types.String `tfsdk:"status"` + PollingOptions *utils.PollingOptions `tfsdk:"polling_options"` } func (p *resourceModel) GetPollingOptions() *utils.PollingOptions { @@ -76,7 +68,7 @@ func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { EbsLLAPSpillGB: p.getEbsLLAPSpillGB(), ImageVersion: p.getImageVersion(), Name: p.Name.ValueStringPointer(), - NodeCount: utils.Int64To32(p.NodeCount), + NodeCount: utils.Int64To32(p.GroupSize), PlatformJwtAuth: p.PlatformJwtAuth.ValueBoolPointer(), QueryIsolationOptions: p.getQueryIsolationOptions(), Autoscaling: p.getAutoscaling(), @@ -122,12 +114,9 @@ func (p *resourceModel) getTags() []*models.TagRequest { } func (p *resourceModel) getQueryIsolationOptions() *models.QueryIsolationOptionsRequest { - if p.QueryIsolationOptions == nil { - return nil - } return &models.QueryIsolationOptionsRequest{ - MaxQueries: utils.Int64To32(p.QueryIsolationOptions.MaxQueries), - MaxNodesPerQuery: utils.Int64To32(p.QueryIsolationOptions.MaxNodesPerQuery), + MaxQueries: utils.Int64To32(p.MaxConcurrentIsolatedQueries), + MaxNodesPerQuery: utils.Int64To32(p.MaxNodesPerIsolatedQuery), } } @@ -146,16 +135,13 @@ func (p *resourceModel) getEbsLLAPSpillGB() int32 { } func (p *resourceModel) getAutoscaling() *models.AutoscalingOptionsCreateRequest { - if p.Autoscaling == nil { - return nil - } return &models.AutoscalingOptionsCreateRequest{ - MinClusters: utils.Int64To32Pointer(p.Autoscaling.MinClusters), - MaxClusters: utils.Int64To32Pointer(p.Autoscaling.MaxClusters), - DisableAutoSuspend: p.Autoscaling.DisableAutoSuspend.ValueBool(), - AutoSuspendTimeoutSeconds: utils.Int64To32(p.Autoscaling.AutoSuspendTimeoutSeconds), - HiveScaleWaitTimeSeconds: utils.Int64To32(p.Autoscaling.HiveScaleWaitTimeSeconds), - HiveDesiredFreeCapacity: utils.Int64To32(p.Autoscaling.HiveDesiredFreeCapacity), + MinClusters: utils.Int64To32Pointer(p.MinGroupCount), + MaxClusters: utils.Int64To32Pointer(p.MaxGroupCount), + DisableAutoSuspend: p.DisableAutoSuspend.ValueBool(), + AutoSuspendTimeoutSeconds: utils.Int64To32(p.AutoSuspendTimeoutSeconds), + HiveScaleWaitTimeSeconds: utils.Int64To32(p.ScaleWaitTimeSeconds), + HiveDesiredFreeCapacity: utils.Int64To32(p.Headroom), } } diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go index 0a4fc976..a234f5e0 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go @@ -69,9 +69,7 @@ func TestAccHive_basic(t *testing.T) { resource.TestCheckResourceAttr("cdp_dw_vw_hive.test_hive", "database_catalog_id", params.DatabaseCatalogID), resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "compactor"), resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jdbc_url"), - resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "kerberos_jdbc_url"), resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "hue_url"), - resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jwt_connection_string"), resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jwt_token_gen_url"), ), }, @@ -86,16 +84,16 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { cluster_id = %[1]q database_catalog_id = %[2]q name = %[3]q - node_count = 2 + group_size = 2 platform_jwt_auth = true enable_sso = true - autoscaling = { - min_clusters = 2 - max_clusters = 5 - disable_auto_suspend = false - auto_suspend_timeout_seconds = 100 - hive_scale_wait_time_seconds = 230 - } + min_group_count = 2 + max_group_count = 5 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + scale_wait_time_seconds = 230 + max_concurrent_isolated_queries = 10 + max_nodes_per_isolated_query = 10 aws_options = { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 @@ -103,10 +101,6 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { owner = "cdw-terraform@cloudera.com" } } - query_isolation_options = { - max_queries = 100 - max_nodes_per_query = 10 - } } `, params.ClusterID, params.DatabaseCatalogID, params.Name) } diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go index c8e3c617..86225de0 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go @@ -69,9 +69,9 @@ var testHiveSchema = schema.Schema{ Computed: true, MarkdownDescription: "The version of the Hive Virtual Warehouse image.", }, - "node_count": schema.Int64Attribute{ + "group_size": schema.Int64Attribute{ Optional: true, - MarkdownDescription: "Nodes per compute cluster. If specified, forces ‘template’ to be ‘custom’.", + MarkdownDescription: "Nodes per compute group. If specified, forces ‘template’ to be ‘custom’.", }, "platform_jwt_auth": schema.BoolAttribute{ Optional: true, @@ -114,37 +114,39 @@ var testHiveSchema = schema.Schema{ Computed: true, MarkdownDescription: "URL to generate JWT tokens for the Virtual Warehouse by the CDP JWT token provider. Available if platform JWT authentication is enabled.", }, - "autoscaling": schema.SingleNestedAttribute{ - MarkdownDescription: "Autoscaling related configuration options that could specify various values that will be used during CDW resource creation.", + "min_group_count": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Minimum number of available compute groups.", + }, + "max_group_count": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Maximum number of available compute groups.", + }, + "disable_auto_suspend": schema.BoolAttribute{ Optional: true, - Attributes: map[string]schema.Attribute{ - "min_clusters": schema.Int64Attribute{ - Required: true, - MarkdownDescription: "Minimum number of available compute groups.", - }, - "max_clusters": schema.Int64Attribute{ - Required: true, - MarkdownDescription: "Maximum number of available compute groups.", - }, - "disable_auto_suspend": schema.BoolAttribute{ - Optional: true, - Computed: true, - Default: booldefault.StaticBool(false), - MarkdownDescription: "Boolean value that specifies if auto-suspend should be disabled.", - }, - "auto_suspend_timeout_seconds": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "The time in seconds after which the compute group should be suspended.", - }, - "hive_scale_wait_time_seconds": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", - }, - "hive_desired_free_capacity": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Set Desired free capacity. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", - }, - }, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Boolean value that specifies if auto-suspend should be disabled.", + }, + "auto_suspend_timeout_seconds": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "The time in seconds after which the compute group should be suspended.", + }, + "scale_wait_time_seconds": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set wait time before a scale event happens. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + }, + "headroom": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + }, + "max_concurrent_isolated_queries": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", + }, + "max_nodes_per_isolated_query": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", }, "aws_options": schema.SingleNestedAttribute{ MarkdownDescription: "AWS related configuration options that could specify various values that will be used during CDW resource creation.", @@ -166,20 +168,6 @@ var testHiveSchema = schema.Schema{ }, }, }, - "query_isolation_options": schema.SingleNestedAttribute{ - MarkdownDescription: "Query isolation related configuration options.", - Optional: true, - Attributes: map[string]schema.Attribute{ - "max_queries": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", - }, - "max_nodes_per_query": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", - }, - }, - }, "last_updated": schema.StringAttribute{ Computed: true, MarkdownDescription: "Timestamp of the last Terraform update of the order.", @@ -240,28 +228,26 @@ func createRawHiveResource() tftypes.Value { "database_catalog_id": tftypes.String, "name": tftypes.String, "image_version": tftypes.String, - "node_count": tftypes.Number, + "group_size": tftypes.Number, "platform_jwt_auth": tftypes.Bool, "ldap_groups": tftypes.List{ ElementType: tftypes.String, }, - "enable_sso": tftypes.Bool, - "compactor": tftypes.Bool, - "jdbc_url": tftypes.String, - "kerberos_jdbc_url": tftypes.String, - "hue_url": tftypes.String, - "jwt_connection_string": tftypes.String, - "jwt_token_gen_url": tftypes.String, - "autoscaling": tftypes.Object{ - AttributeTypes: map[string]tftypes.Type{ - "min_clusters": tftypes.Number, - "max_clusters": tftypes.Number, - "disable_auto_suspend": tftypes.Bool, - "auto_suspend_timeout_seconds": tftypes.Number, - "hive_scale_wait_time_seconds": tftypes.Number, - "hive_desired_free_capacity": tftypes.Number, - }, - }, + "enable_sso": tftypes.Bool, + "compactor": tftypes.Bool, + "jdbc_url": tftypes.String, + "kerberos_jdbc_url": tftypes.String, + "hue_url": tftypes.String, + "jwt_connection_string": tftypes.String, + "jwt_token_gen_url": tftypes.String, + "min_group_count": tftypes.Number, + "max_group_count": tftypes.Number, + "disable_auto_suspend": tftypes.Bool, + "auto_suspend_timeout_seconds": tftypes.Number, + "scale_wait_time_seconds": tftypes.Number, + "headroom": tftypes.Number, + "max_concurrent_isolated_queries": tftypes.Number, + "max_nodes_per_isolated_query": tftypes.Number, "aws_options": tftypes.Object{ AttributeTypes: map[string]tftypes.Type{ "availability_zone": tftypes.String, @@ -269,12 +255,6 @@ func createRawHiveResource() tftypes.Value { "tags": tftypes.Map{ElementType: tftypes.String}, }, }, - "query_isolation_options": tftypes.Object{ - AttributeTypes: map[string]tftypes.Type{ - "max_queries": tftypes.Number, - "max_nodes_per_query": tftypes.Number, - }, - }, "last_updated": tftypes.String, "status": tftypes.String, "polling_options": tftypes.Object{ @@ -286,38 +266,29 @@ func createRawHiveResource() tftypes.Value { }, }}, map[string]tftypes.Value{ - "id": tftypes.NewValue(tftypes.String, ""), - "cluster_id": tftypes.NewValue(tftypes.String, "cluster-id"), - "database_catalog_id": tftypes.NewValue(tftypes.String, "database-catalog-id"), - "name": tftypes.NewValue(tftypes.String, ""), - "image_version": tftypes.NewValue(tftypes.String, ""), - "node_count": tftypes.NewValue(tftypes.Number, 10), - "platform_jwt_auth": tftypes.NewValue(tftypes.Bool, true), - "ldap_groups": tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{}), - "enable_sso": tftypes.NewValue(tftypes.Bool, true), - "compactor": tftypes.NewValue(tftypes.Bool, false), - "jdbc_url": tftypes.NewValue(tftypes.String, ""), - "kerberos_jdbc_url": tftypes.NewValue(tftypes.String, ""), - "hue_url": tftypes.NewValue(tftypes.String, ""), - "jwt_connection_string": tftypes.NewValue(tftypes.String, ""), - "jwt_token_gen_url": tftypes.NewValue(tftypes.String, ""), - "autoscaling": tftypes.NewValue( - tftypes.Object{ - AttributeTypes: map[string]tftypes.Type{ - "min_clusters": tftypes.Number, - "max_clusters": tftypes.Number, - "disable_auto_suspend": tftypes.Bool, - "auto_suspend_timeout_seconds": tftypes.Number, - "hive_scale_wait_time_seconds": tftypes.Number, - "hive_desired_free_capacity": tftypes.Number, - }}, map[string]tftypes.Value{ - "min_clusters": tftypes.NewValue(tftypes.Number, 1), - "max_clusters": tftypes.NewValue(tftypes.Number, 10), - "disable_auto_suspend": tftypes.NewValue(tftypes.Bool, false), - "auto_suspend_timeout_seconds": tftypes.NewValue(tftypes.Number, 60), - "hive_scale_wait_time_seconds": tftypes.NewValue(tftypes.Number, 60), - "hive_desired_free_capacity": tftypes.NewValue(tftypes.Number, 10), - }), + "id": tftypes.NewValue(tftypes.String, ""), + "cluster_id": tftypes.NewValue(tftypes.String, "cluster-id"), + "database_catalog_id": tftypes.NewValue(tftypes.String, "database-catalog-id"), + "name": tftypes.NewValue(tftypes.String, ""), + "image_version": tftypes.NewValue(tftypes.String, ""), + "group_size": tftypes.NewValue(tftypes.Number, 10), + "platform_jwt_auth": tftypes.NewValue(tftypes.Bool, true), + "ldap_groups": tftypes.NewValue(tftypes.List{ElementType: tftypes.String}, []tftypes.Value{}), + "enable_sso": tftypes.NewValue(tftypes.Bool, true), + "compactor": tftypes.NewValue(tftypes.Bool, false), + "jdbc_url": tftypes.NewValue(tftypes.String, ""), + "kerberos_jdbc_url": tftypes.NewValue(tftypes.String, ""), + "hue_url": tftypes.NewValue(tftypes.String, ""), + "jwt_connection_string": tftypes.NewValue(tftypes.String, ""), + "jwt_token_gen_url": tftypes.NewValue(tftypes.String, ""), + "min_group_count": tftypes.NewValue(tftypes.Number, 1), + "max_group_count": tftypes.NewValue(tftypes.Number, 10), + "disable_auto_suspend": tftypes.NewValue(tftypes.Bool, false), + "auto_suspend_timeout_seconds": tftypes.NewValue(tftypes.Number, 60), + "scale_wait_time_seconds": tftypes.NewValue(tftypes.Number, 60), + "headroom": tftypes.NewValue(tftypes.Number, 10), + "max_concurrent_isolated_queries": tftypes.NewValue(tftypes.Number, 10), + "max_nodes_per_isolated_query": tftypes.NewValue(tftypes.Number, 10), "aws_options": tftypes.NewValue( tftypes.Object{ AttributeTypes: map[string]tftypes.Type{ @@ -332,15 +303,6 @@ func createRawHiveResource() tftypes.Value { "owner": tftypes.NewValue(tftypes.String, "dw-terraform@cloudera.com"), }), }), - "query_isolation_options": tftypes.NewValue( - tftypes.Object{ - AttributeTypes: map[string]tftypes.Type{ - "max_queries": tftypes.Number, - "max_nodes_per_query": tftypes.Number, - }}, map[string]tftypes.Value{ - "max_queries": tftypes.NewValue(tftypes.Number, 10), - "max_nodes_per_query": tftypes.NewValue(tftypes.Number, 10), - }), "last_updated": tftypes.NewValue(tftypes.String, ""), "status": tftypes.NewValue(tftypes.String, "Running"), "polling_options": tftypes.NewValue( @@ -595,31 +557,27 @@ func (suite *HiveTestSuite) TestStateRefresh_FailureThresholdReached() { func (suite *HiveTestSuite) TestConvertToCreateVwRequest_All() { plan := resourceModel{ - ClusterID: types.StringValue("cluster-id"), - DatabaseCatalogID: types.StringValue("database-catalog-id"), - Name: types.StringValue("test-name"), - ImageVersion: types.StringValue("2024.0.19.0-301"), - NodeCount: types.Int64Value(10), - PlatformJwtAuth: types.BoolValue(true), - LdapGroups: types.ListValueMust(types.StringType, []attr.Value{types.StringValue("ldap-group")}), - EnableSSO: types.BoolValue(true), - Autoscaling: &autoscaling{ - MinClusters: types.Int64Value(1), - MaxClusters: types.Int64Value(10), - DisableAutoSuspend: types.BoolValue(false), - AutoSuspendTimeoutSeconds: types.Int64Value(60), - HiveScaleWaitTimeSeconds: types.Int64Value(60), - HiveDesiredFreeCapacity: types.Int64Value(2), - }, + ClusterID: types.StringValue("cluster-id"), + DatabaseCatalogID: types.StringValue("database-catalog-id"), + Name: types.StringValue("test-name"), + ImageVersion: types.StringValue("2024.0.19.0-301"), + GroupSize: types.Int64Value(10), + PlatformJwtAuth: types.BoolValue(true), + LdapGroups: types.ListValueMust(types.StringType, []attr.Value{types.StringValue("ldap-group")}), + EnableSSO: types.BoolValue(true), + MinGroupCount: types.Int64Value(1), + MaxGroupCount: types.Int64Value(10), + DisableAutoSuspend: types.BoolValue(false), + AutoSuspendTimeoutSeconds: types.Int64Value(60), + ScaleWaitTimeSeconds: types.Int64Value(60), + Headroom: types.Int64Value(2), + MaxConcurrentIsolatedQueries: types.Int64Value(5), + MaxNodesPerIsolatedQuery: types.Int64Value(2), AwsOptions: &awsOptions{ AvailabilityZone: types.StringValue("us-west-2a"), EbsLLAPSpillGb: types.Int64Value(300), Tags: types.MapValueMust(types.StringType, map[string]attr.Value{"key1": types.StringValue("value1")}), }, - QueryIsolationOptions: &queryIsolationOptions{ - MaxQueries: types.Int64Value(5), - MaxNodesPerQuery: types.Int64Value(2), - }, } req := plan.convertToCreateVwRequest() @@ -649,7 +607,6 @@ func (suite *HiveTestSuite) TestConvertToCreateVwRequest_MissingImageVersion() { plan := resourceModel{ ImageVersion: types.StringUnknown(), AwsOptions: &awsOptions{}, - Autoscaling: &autoscaling{}, } req := plan.convertToCreateVwRequest() diff --git a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go index 059525bb..bffdd468 100644 --- a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go @@ -52,9 +52,9 @@ var hiveSchema = schema.Schema{ Computed: true, MarkdownDescription: "The version of the Hive Virtual Warehouse image.", }, - "node_count": schema.Int64Attribute{ + "group_size": schema.Int64Attribute{ Optional: true, - MarkdownDescription: "Nodes per compute cluster. If specified, forces ‘template’ to be ‘custom’.", + MarkdownDescription: "Nodes per compute group. If specified, forces ‘template’ to be ‘custom’.", }, "platform_jwt_auth": schema.BoolAttribute{ Optional: true, @@ -97,37 +97,39 @@ var hiveSchema = schema.Schema{ Computed: true, MarkdownDescription: "URL to generate JWT tokens for the Virtual Warehouse by the CDP JWT token provider. Available if platform JWT authentication is enabled.", }, - "autoscaling": schema.SingleNestedAttribute{ - MarkdownDescription: "Autoscaling related configuration options that could specify various values that will be used during CDW resource creation.", + "min_group_count": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Minimum number of available compute groups.", + }, + "max_group_count": schema.Int64Attribute{ + Required: true, + MarkdownDescription: "Maximum number of available compute groups.", + }, + "disable_auto_suspend": schema.BoolAttribute{ Optional: true, - Attributes: map[string]schema.Attribute{ - "min_clusters": schema.Int64Attribute{ - Required: true, - MarkdownDescription: "Minimum number of available compute groups.", - }, - "max_clusters": schema.Int64Attribute{ - Required: true, - MarkdownDescription: "Maximum number of available compute groups.", - }, - "disable_auto_suspend": schema.BoolAttribute{ - Optional: true, - Computed: true, - Default: booldefault.StaticBool(false), - MarkdownDescription: "Boolean value that specifies if auto-suspend should be disabled.", - }, - "auto_suspend_timeout_seconds": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "The time in seconds after which the compute group should be suspended.", - }, - "hive_scale_wait_time_seconds": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", - }, - "hive_desired_free_capacity": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Set Desired free capacity. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided.", - }, - }, + Computed: true, + Default: booldefault.StaticBool(false), + MarkdownDescription: "Boolean value that specifies if auto-suspend should be disabled.", + }, + "auto_suspend_timeout_seconds": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "The time in seconds after which the compute group should be suspended.", + }, + "scale_wait_time_seconds": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set wait time before a scale event happens. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + }, + "headroom": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + }, + "max_concurrent_isolated_queries": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", + }, + "max_nodes_per_isolated_query": schema.Int64Attribute{ + Optional: true, + MarkdownDescription: "Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", }, "aws_options": schema.SingleNestedAttribute{ MarkdownDescription: "AWS related configuration options that could specify various values that will be used during CDW resource creation.", @@ -149,20 +151,6 @@ var hiveSchema = schema.Schema{ }, }, }, - "query_isolation_options": schema.SingleNestedAttribute{ - MarkdownDescription: "Query isolation related configuration options.", - Optional: true, - Attributes: map[string]schema.Attribute{ - "max_queries": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", - }, - "max_nodes_per_query": schema.Int64Attribute{ - Optional: true, - MarkdownDescription: "Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled.", - }, - }, - }, "last_updated": schema.StringAttribute{ Computed: true, MarkdownDescription: "Timestamp of the last Terraform update of the order.", From bacb1f391f8da5a723828640de7daf2c3a05612e Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Mon, 16 Dec 2024 16:14:20 -0600 Subject: [PATCH 8/9] Add validation to make headroom and scale wait time properties conflicting Either headroom or scale wait time should be set --- docs/resources/dw_vw_hive.md | 81 +++++++++---------- examples/resources/cdp_dw_vw_hive/resource.tf | 44 ++++++---- go.mod | 21 ++--- go.sum | 34 ++++---- resources/dw/resource_dw_acc_test.go | 32 ++++---- .../virtualwarehouse/hive/resource_hive_vw.go | 10 +++ .../hive/resource_hive_vw_acc_test.go | 4 +- .../hive/resource_hive_vw_test.go | 44 +++++++--- .../virtualwarehouse/hive/schema_hive_vw.go | 6 +- 9 files changed, 165 insertions(+), 111 deletions(-) diff --git a/docs/resources/dw_vw_hive.md b/docs/resources/dw_vw_hive.md index 9c2978a1..47359d3a 100644 --- a/docs/resources/dw_vw_hive.md +++ b/docs/resources/dw_vw_hive.md @@ -26,18 +26,18 @@ resource "cdp_dw_vw_hive" "example" { cluster_id = "env-id" database_catalog_id = "warehouse-id" name = "default-catalog" - node_count = 2 + group_size = 2 platform_jwt_auth = true enable_sso = true - image_version = "2024.0.19.0-301" - autoscaling = { - min_clusters = 1 - max_clusters = 3 - disable_auto_suspend = false - auto_suspend_timeout_seconds = 100 - hive_scale_wait_time_seconds = 230 - hive_desired_free_capacity = 1 - } + image_version = "2024.0.18.4-5" + min_group_count = 1 + max_group_count = 3 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + scale_wait_time_seconds = 230 // either headroom or scale_wait_time_seconds can be configured + headroom = 1 + max_concurrent_isolated_queries = 5 + max_nodes_per_isolated_query = 2 aws_options = { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 @@ -45,10 +45,26 @@ resource "cdp_dw_vw_hive" "example" { "key1" = "value1" } } - query_isolation_options = { - max_queries = 100 - max_nodes_per_query = 10 - } +} + +output "jdbc_url" { + value = cdp_dw_vw_hive.example.jdbc_url +} + +output "kerberos_jdbc_url" { + value = cdp_dw_vw_hive.example.kerberos_jdbc_url +} + +output "hue_url" { + value = cdp_dw_vw_hive.example.hue_url +} + +output "jwt_connection_string" { + value = cdp_dw_vw_hive.example.jwt_connection_string +} + +output "jwt_token_gen_url" { + value = cdp_dw_vw_hive.example.jwt_token_gen_url } ``` @@ -59,19 +75,25 @@ resource "cdp_dw_vw_hive" "example" { - `cluster_id` (String) The id of the CDW Cluster which the Hive Virtual Warehouse is attached to. - `database_catalog_id` (String) The id of the Database Catalog which the Hive Virtual Warehouse is attached to. +- `group_size` (Number) Nodes per compute group. If specified, forces ‘template’ to be ‘custom’. +- `max_group_count` (Number) Maximum number of available compute groups. +- `min_group_count` (Number) Minimum number of available compute groups. - `name` (String) The name of the Hive Virtual Warehouse. ### Optional -- `autoscaling` (Attributes) Autoscaling related configuration options that could specify various values that will be used during CDW resource creation. (see [below for nested schema](#nestedatt--autoscaling)) +- `auto_suspend_timeout_seconds` (Number) The time in seconds after which the compute group should be suspended. - `aws_options` (Attributes) AWS related configuration options that could specify various values that will be used during CDW resource creation. (see [below for nested schema](#nestedatt--aws_options)) +- `disable_auto_suspend` (Boolean) Boolean value that specifies if auto-suspend should be disabled. - `enable_sso` (Boolean) Enable SSO for the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. +- `headroom` (Number) Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs. - `image_version` (String) The version of the Hive Virtual Warehouse image. - `ldap_groups` (List of String) LDAP group names to be enabled for auth. -- `node_count` (Number) Nodes per compute cluster. If specified, forces ‘template’ to be ‘custom’. +- `max_concurrent_isolated_queries` (Number) Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. +- `max_nodes_per_isolated_query` (Number) Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. - `platform_jwt_auth` (Boolean) Value of ‘true’ automatically configures the Virtual Warehouse to support JWTs issued by the CDP JWT token provider. Value of ‘false’ does not enable JWT auth on the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. - `polling_options` (Attributes) Polling related configuration options that could specify various values that will be used during CDP resource creation. (see [below for nested schema](#nestedatt--polling_options)) -- `query_isolation_options` (Attributes) Query isolation related configuration options. (see [below for nested schema](#nestedatt--query_isolation_options)) +- `scale_wait_time_seconds` (Number) Set wait time before a scale event happens. ### Read-Only @@ -85,22 +107,6 @@ resource "cdp_dw_vw_hive" "example" { - `last_updated` (String) Timestamp of the last Terraform update of the order. - `status` (String) The status of the database catalog. - -### Nested Schema for `autoscaling` - -Required: - -- `max_clusters` (Number) Maximum number of available compute groups. -- `min_clusters` (Number) Minimum number of available compute groups. - -Optional: - -- `auto_suspend_timeout_seconds` (Number) The time in seconds after which the compute group should be suspended. -- `disable_auto_suspend` (Boolean) Boolean value that specifies if auto-suspend should be disabled. -- `hive_desired_free_capacity` (Number) Set Desired free capacity. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided. -- `hive_scale_wait_time_seconds` (Number) Set wait time before a scale event happens. Either “hiveScaleWaitTimeSeconds” or “hiveDesiredFreeCapacity” can be provided. - - ### Nested Schema for `aws_options` @@ -119,12 +125,3 @@ Optional: - `async` (Boolean) Boolean value that specifies if Terraform should wait for resource creation/deletion. - `call_failure_threshold` (Number) Threshold value that specifies how many times should a single call failure happen before giving up the polling. - `polling_timeout` (Number) Timeout value in minutes that specifies for how long should the polling go for resource creation/deletion. - - - -### Nested Schema for `query_isolation_options` - -Optional: - -- `max_nodes_per_query` (Number) Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. -- `max_queries` (Number) Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. diff --git a/examples/resources/cdp_dw_vw_hive/resource.tf b/examples/resources/cdp_dw_vw_hive/resource.tf index 04b75c76..f6d434fe 100644 --- a/examples/resources/cdp_dw_vw_hive/resource.tf +++ b/examples/resources/cdp_dw_vw_hive/resource.tf @@ -12,18 +12,18 @@ resource "cdp_dw_vw_hive" "example" { cluster_id = "env-id" database_catalog_id = "warehouse-id" name = "default-catalog" - node_count = 2 + group_size = 2 platform_jwt_auth = true enable_sso = true - image_version = "2024.0.19.0-301" - autoscaling = { - min_clusters = 1 - max_clusters = 3 - disable_auto_suspend = false - auto_suspend_timeout_seconds = 100 - hive_scale_wait_time_seconds = 230 - hive_desired_free_capacity = 1 - } + image_version = "2024.0.18.4-5" + min_group_count = 1 + max_group_count = 3 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + scale_wait_time_seconds = 230 // either headroom or scale_wait_time_seconds can be configured + headroom = 1 + max_concurrent_isolated_queries = 5 + max_nodes_per_isolated_query = 2 aws_options = { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 @@ -31,8 +31,24 @@ resource "cdp_dw_vw_hive" "example" { "key1" = "value1" } } - query_isolation_options = { - max_queries = 100 - max_nodes_per_query = 10 - } } + +output "jdbc_url" { + value = cdp_dw_vw_hive.example.jdbc_url +} + +output "kerberos_jdbc_url" { + value = cdp_dw_vw_hive.example.kerberos_jdbc_url +} + +output "hue_url" { + value = cdp_dw_vw_hive.example.hue_url +} + +output "jwt_connection_string" { + value = cdp_dw_vw_hive.example.jwt_connection_string +} + +output "jwt_token_gen_url" { + value = cdp_dw_vw_hive.example.jwt_token_gen_url +} \ No newline at end of file diff --git a/go.mod b/go.mod index 339235ed..fa09146d 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/cloudera/terraform-provider-cdp -go 1.21 +go 1.22.0 -toolchain go1.22.1 +toolchain go1.22.10 require ( github.com/go-openapi/errors v0.22.0 @@ -12,8 +12,9 @@ require ( github.com/go-openapi/validate v0.24.0 github.com/google/uuid v1.6.0 github.com/hashicorp/terraform-plugin-docs v0.14.1 - github.com/hashicorp/terraform-plugin-framework v1.7.0 - github.com/hashicorp/terraform-plugin-go v0.22.1 + github.com/hashicorp/terraform-plugin-framework v1.13.0 + github.com/hashicorp/terraform-plugin-framework-validators v0.16.0 + github.com/hashicorp/terraform-plugin-go v0.25.0 github.com/hashicorp/terraform-plugin-log v0.9.0 github.com/hashicorp/terraform-plugin-sdk/v2 v2.33.0 github.com/hashicorp/terraform-plugin-testing v1.7.0 @@ -22,7 +23,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.9.0 golang.org/x/crypto v0.31.0 - golang.org/x/net v0.25.0 + golang.org/x/net v0.28.0 golang.org/x/text v0.21.0 gopkg.in/ini.v1 v1.67.0 ) @@ -47,7 +48,7 @@ require ( github.com/go-openapi/jsonreference v0.21.0 // indirect github.com/go-openapi/loads v0.22.0 // indirect github.com/go-openapi/spec v0.21.0 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-checkpoint v0.5.0 // indirect @@ -55,7 +56,7 @@ require ( github.com/hashicorp/go-cty v1.4.1-0.20200414143053-d3edf31b6320 // indirect github.com/hashicorp/go-hclog v1.6.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-plugin v1.6.0 // indirect + github.com/hashicorp/go-plugin v1.6.2 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hc-install v0.6.3 // indirect @@ -101,8 +102,8 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect - google.golang.org/grpc v1.62.1 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/grpc v1.67.1 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index e62d78a6..ddebd36b 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -106,8 +106,8 @@ github.com/hashicorp/go-hclog v1.6.2/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVH github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-plugin v1.6.0 h1:wgd4KxHJTVGGqWBq4QPB1i5BZNEx9BR8+OFmHDmTk8A= -github.com/hashicorp/go-plugin v1.6.0/go.mod h1:lBS5MtSSBZk0SHc66KACcjjlU6WzEVP/8pwz68aMkCI= +github.com/hashicorp/go-plugin v1.6.2 h1:zdGAEd0V1lCaU0u+MxWQhtSDQmahpkwOun8U8EiRVog= +github.com/hashicorp/go-plugin v1.6.2/go.mod h1:CkgLQ5CZqNmdL9U9JzM532t8ZiYQ35+pj3b1FD37R0Q= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -125,10 +125,12 @@ github.com/hashicorp/terraform-json v0.21.0 h1:9NQxbLNqPbEMze+S6+YluEdXgJmhQykRy github.com/hashicorp/terraform-json v0.21.0/go.mod h1:qdeBs11ovMzo5puhrRibdD6d2Dq6TyE/28JiU4tIQxk= github.com/hashicorp/terraform-plugin-docs v0.14.1 h1:MikFi59KxrP/ewrZoaowrB9he5Vu4FtvhamZFustiA4= github.com/hashicorp/terraform-plugin-docs v0.14.1/go.mod h1:k2NW8+t113jAus6bb5tQYQgEAX/KueE/u8X2Z45V1GM= -github.com/hashicorp/terraform-plugin-framework v1.7.0 h1:wOULbVmfONnJo9iq7/q+iBOBJul5vRovaYJIu2cY/Pw= -github.com/hashicorp/terraform-plugin-framework v1.7.0/go.mod h1:jY9Id+3KbZ17OMpulgnWLSfwxNVYSoYBQFTgsx044CI= -github.com/hashicorp/terraform-plugin-go v0.22.1 h1:iTS7WHNVrn7uhe3cojtvWWn83cm2Z6ryIUDTRO0EV7w= -github.com/hashicorp/terraform-plugin-go v0.22.1/go.mod h1:qrjnqRghvQ6KnDbB12XeZ4FluclYwptntoWCr9QaXTI= +github.com/hashicorp/terraform-plugin-framework v1.13.0 h1:8OTG4+oZUfKgnfTdPTJwZ532Bh2BobF4H+yBiYJ/scw= +github.com/hashicorp/terraform-plugin-framework v1.13.0/go.mod h1:j64rwMGpgM3NYXTKuxrCnyubQb/4VKldEKlcG8cvmjU= +github.com/hashicorp/terraform-plugin-framework-validators v0.16.0 h1:O9QqGoYDzQT7lwTXUsZEtgabeWW96zUBh47Smn2lkFA= +github.com/hashicorp/terraform-plugin-framework-validators v0.16.0/go.mod h1:Bh89/hNmqsEWug4/XWKYBwtnw3tbz5BAy1L1OgvbIaY= +github.com/hashicorp/terraform-plugin-go v0.25.0 h1:oi13cx7xXA6QciMcpcFi/rwA974rdTxjqEhXJjbAyks= +github.com/hashicorp/terraform-plugin-go v0.25.0/go.mod h1:+SYagMYadJP86Kvn+TGeV+ofr/R3g4/If0O5sO96MVw= github.com/hashicorp/terraform-plugin-log v0.9.0 h1:i7hOA+vdAItN1/7UrfBqBwvYPQ9TFvymaRGZED3FCV0= github.com/hashicorp/terraform-plugin-log v0.9.0/go.mod h1:rKL8egZQ/eXSyDqzLUuwUYLVdlYeamldAHSxjUFADow= github.com/hashicorp/terraform-plugin-sdk/v2 v2.33.0 h1:qHprzXy/As0rxedphECBEQAh3R4yp6pKksKHcqZx5G8= @@ -277,8 +279,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -323,14 +325,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/resources/dw/resource_dw_acc_test.go b/resources/dw/resource_dw_acc_test.go index dc2a073b..e15005f6 100644 --- a/resources/dw/resource_dw_acc_test.go +++ b/resources/dw/resource_dw_acc_test.go @@ -143,7 +143,6 @@ func TestAccDwCluster_Basic(t *testing.T) { StorageLocationBase: os.Getenv(AwsStorageLocationBase), Runtime: os.Getenv(AwsRuntime), } - resourceName := "cdp_dw_aws_cluster.test_data_warehouse_aws" resource.Test(t, resource.TestCase{ PreCheck: func() { cdpacctest.PreCheck(t) @@ -168,8 +167,12 @@ func TestAccDwCluster_Basic(t *testing.T) { testAccDwCatalog(), testAccHiveVirtualWarehouse(cdpacctest.RandomShortWithPrefix("tf-hive"))), Check: resource.ComposeAggregateTestCheckFunc( - resource.TestCheckResourceAttr(resourceName, "name", envParams.Name), - resource.TestCheckResourceAttr(resourceName, "status", "Accepted"), + resource.TestCheckResourceAttr("cdp_dw_aws_cluster.test_data_warehouse_aws", "name", envParams.Name), + resource.TestCheckResourceAttr("cdp_dw_aws_cluster.test_data_warehouse_aws", "status", "Accepted"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "compactor"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jdbc_url"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "hue_url"), + resource.TestCheckResourceAttrSet("cdp_dw_vw_hive.test_hive", "jwt_token_gen_url"), ), }, // Delete testing automatically occurs in TestCase @@ -283,24 +286,23 @@ func testAccHiveVirtualWarehouse(name string) string { cluster_id = cdp_dw_aws_cluster.test_data_warehouse_aws.cluster_id database_catalog_id = cdp_dw_database_catalog.test_catalog.id name = %[1]q + group_size = 2 platform_jwt_auth = true enable_sso = true - image_version = "2024.0.19.0-301" - node_count = 2 - autoscaling = { - min_clusters = 1 - max_clusters = 3 - disable_auto_suspend = false - auto_suspend_timeout_seconds = 100 - hive_scale_wait_time_seconds = 230 - } - aws_options = { + min_group_count = 2 + max_group_count = 5 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 + scale_wait_time_seconds = 230 + max_concurrent_isolated_queries = 10 + max_nodes_per_isolated_query = 10 + aws_options = { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 tags = { - owner = "cdw-terraform" + owner = "cdw-terraform@cloudera.com" } - } + } } `, name) } diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go index 69aeb03f..e4b9d29a 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go @@ -16,6 +16,7 @@ import ( "strings" "time" + "github.com/hashicorp/terraform-plugin-framework-validators/resourcevalidator" "github.com/hashicorp/terraform-plugin-framework/path" "github.com/hashicorp/terraform-plugin-framework/resource" "github.com/hashicorp/terraform-plugin-log/tflog" @@ -57,6 +58,15 @@ func (r *hiveResource) Schema(_ context.Context, _ resource.SchemaRequest, resp resp.Schema = hiveSchema } +func (r *hiveResource) ConfigValidators(context.Context) []resource.ConfigValidator { + return []resource.ConfigValidator{ + resourcevalidator.Conflicting( + path.MatchRoot("scale_wait_time_seconds"), + path.MatchRoot("headroom"), + ), + } +} + func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, resp *resource.CreateResponse) { var plan resourceModel diags := req.Plan.Get(ctx, &plan) diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go index a234f5e0..9935eea5 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_acc_test.go @@ -98,8 +98,8 @@ func testAccHiveBasicConfig(params hiveTestParameters) string { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 tags = { - owner = "cdw-terraform@cloudera.com" - } + owner = "cdw-terraform@cloudera.com" + } } } `, params.ClusterID, params.DatabaseCatalogID, params.Name) diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go index 86225de0..18f7fd10 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go @@ -17,6 +17,7 @@ import ( "github.com/go-openapi/runtime" "github.com/hashicorp/terraform-plugin-framework/attr" + "github.com/hashicorp/terraform-plugin-framework/diag" "github.com/hashicorp/terraform-plugin-framework/resource" "github.com/hashicorp/terraform-plugin-framework/resource/schema" "github.com/hashicorp/terraform-plugin-framework/resource/schema/booldefault" @@ -70,7 +71,7 @@ var testHiveSchema = schema.Schema{ MarkdownDescription: "The version of the Hive Virtual Warehouse image.", }, "group_size": schema.Int64Attribute{ - Optional: true, + Required: true, MarkdownDescription: "Nodes per compute group. If specified, forces ‘template’ to be ‘custom’.", }, "platform_jwt_auth": schema.BoolAttribute{ @@ -134,11 +135,11 @@ var testHiveSchema = schema.Schema{ }, "scale_wait_time_seconds": schema.Int64Attribute{ Optional: true, - MarkdownDescription: "Set wait time before a scale event happens. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + MarkdownDescription: "Set wait time before a scale event happens.", }, "headroom": schema.Int64Attribute{ Optional: true, - MarkdownDescription: "Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + MarkdownDescription: "Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs.", }, "max_concurrent_isolated_queries": schema.Int64Attribute{ Optional: true, @@ -409,6 +410,31 @@ func (suite *HiveTestSuite) TestHiveCreate_Success() { suite.Equal("test-name", result.Name.ValueString()) } +func (suite *HiveTestSuite) TestHiveValidate_Headroom_ScaleWaitTime() { + ctx := context.TODO() + client := new(mocks.MockDwClientService) + dwApi := newDwApi(client) + + req := resource.ValidateConfigRequest{ + Config: tfsdk.Config{ + Raw: createRawHiveResource(), + Schema: testHiveSchema, + }, + } + + resp := &resource.ValidateConfigResponse{ + Diagnostics: diag.Diagnostics{}, + } + + // Function under test + validator := dwApi.ConfigValidators(ctx) + for _, v := range validator { + v.ValidateResource(ctx, req, resp) + } + suite.Contains(resp.Diagnostics.Errors()[0].Summary(), "Invalid Attribute Combination") + suite.Contains(resp.Diagnostics.Errors()[0].Detail(), "These attributes cannot be configured together: [scale_wait_time_seconds,headroom]") +} + func (suite *HiveTestSuite) TestHiveCreate_CreationError() { ctx := context.TODO() client := new(mocks.MockDwClientService) @@ -569,8 +595,8 @@ func (suite *HiveTestSuite) TestConvertToCreateVwRequest_All() { MaxGroupCount: types.Int64Value(10), DisableAutoSuspend: types.BoolValue(false), AutoSuspendTimeoutSeconds: types.Int64Value(60), - ScaleWaitTimeSeconds: types.Int64Value(60), Headroom: types.Int64Value(2), + ScaleWaitTimeSeconds: types.Int64Value(60), MaxConcurrentIsolatedQueries: types.Int64Value(5), MaxNodesPerIsolatedQuery: types.Int64Value(2), AwsOptions: &awsOptions{ @@ -585,16 +611,16 @@ func (suite *HiveTestSuite) TestConvertToCreateVwRequest_All() { suite.Equal("database-catalog-id", *req.DbcID) suite.Equal("test-name", *req.Name) suite.Equal("2024.0.19.0-301", req.ImageVersion) - suite.Equal(int32(10), req.NodeCount) + suite.Equal(int32(10), req.NodeCount) // group_size suite.Equal(true, *req.PlatformJwtAuth) suite.Equal([]string{"ldap-group"}, req.Config.LdapGroups) suite.Equal(true, req.Config.EnableSSO) - suite.Equal(int32(1), *req.Autoscaling.MinClusters) - suite.Equal(int32(10), *req.Autoscaling.MaxClusters) + suite.Equal(int32(1), *req.Autoscaling.MinClusters) // min_group_size + suite.Equal(int32(10), *req.Autoscaling.MaxClusters) // max_group_size suite.Equal(false, req.Autoscaling.DisableAutoSuspend) suite.Equal(int32(60), req.Autoscaling.AutoSuspendTimeoutSeconds) - suite.Equal(int32(60), req.Autoscaling.HiveScaleWaitTimeSeconds) - suite.Equal(int32(2), req.Autoscaling.HiveDesiredFreeCapacity) + suite.Equal(int32(60), req.Autoscaling.HiveScaleWaitTimeSeconds) // scale_wait_time_seconds + suite.Equal(int32(2), req.Autoscaling.HiveDesiredFreeCapacity) // headroom suite.Equal("us-west-2a", req.AvailabilityZone) suite.Equal(int32(300), req.EbsLLAPSpillGB) suite.Equal("key1", *req.Tags[0].Key) diff --git a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go index bffdd468..e9fe8d35 100644 --- a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go @@ -53,7 +53,7 @@ var hiveSchema = schema.Schema{ MarkdownDescription: "The version of the Hive Virtual Warehouse image.", }, "group_size": schema.Int64Attribute{ - Optional: true, + Required: true, MarkdownDescription: "Nodes per compute group. If specified, forces ‘template’ to be ‘custom’.", }, "platform_jwt_auth": schema.BoolAttribute{ @@ -117,11 +117,11 @@ var hiveSchema = schema.Schema{ }, "scale_wait_time_seconds": schema.Int64Attribute{ Optional: true, - MarkdownDescription: "Set wait time before a scale event happens. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + MarkdownDescription: "Set wait time before a scale event happens.", }, "headroom": schema.Int64Attribute{ Optional: true, - MarkdownDescription: "Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs. Either “scale_wait_time_in_seconds” or “headroom” can be provided.", + MarkdownDescription: "Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs.", }, "max_concurrent_isolated_queries": schema.Int64Attribute{ Optional: true, From 67447307c95fdc873e8c25c25b16ffae55452e30 Mon Sep 17 00:00:00 2001 From: Agnes Tevesz Date: Tue, 17 Dec 2024 19:47:43 -0600 Subject: [PATCH 9/9] Addressing review comments --- docs/resources/dw_vw_hive.md | 40 +++++++++++-------- examples/resources/cdp_dw_vw_hive/resource.tf | 36 ++++++++++------- .../dw/virtualwarehouse/hive/model_hive_vw.go | 39 +++++++++--------- .../virtualwarehouse/hive/resource_hive_vw.go | 7 +++- .../hive/resource_hive_vw_test.go | 12 ++++-- .../virtualwarehouse/hive/schema_hive_vw.go | 4 +- 6 files changed, 81 insertions(+), 57 deletions(-) diff --git a/docs/resources/dw_vw_hive.md b/docs/resources/dw_vw_hive.md index 47359d3a..37163c64 100644 --- a/docs/resources/dw_vw_hive.md +++ b/docs/resources/dw_vw_hive.md @@ -22,22 +22,30 @@ A Hive Virtual Warehouse is service which is able to run big SQL queries. # OF ANY KIND, either express or implied. Refer to the License for the specific # permissions and limitations governing your use of the file. +terraform { + required_providers { + cdp = { + source = "cloudera/cdp" + } + } +} + resource "cdp_dw_vw_hive" "example" { - cluster_id = "env-id" - database_catalog_id = "warehouse-id" - name = "default-catalog" - group_size = 2 - platform_jwt_auth = true - enable_sso = true - image_version = "2024.0.18.4-5" - min_group_count = 1 - max_group_count = 3 - disable_auto_suspend = false - auto_suspend_timeout_seconds = 100 + cluster_id = "env-id" + database_catalog_id = "warehouse-id" + name = "default-catalog" + group_size = 2 + platform_jwt_auth = true + enable_sso = true + image_version = "2024.0.18.4-5" + min_group_count = 1 + max_group_count = 3 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 scale_wait_time_seconds = 230 // either headroom or scale_wait_time_seconds can be configured - headroom = 1 + headroom = 1 max_concurrent_isolated_queries = 5 - max_nodes_per_isolated_query = 2 + max_nodes_per_isolated_query = 2 aws_options = { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 @@ -64,7 +72,7 @@ output "jwt_connection_string" { } output "jwt_token_gen_url" { - value = cdp_dw_vw_hive.example.jwt_token_gen_url + value = cdp_dw_vw_hive.example.jwt_token_gen_url } ``` @@ -88,7 +96,7 @@ output "jwt_token_gen_url" { - `enable_sso` (Boolean) Enable SSO for the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. - `headroom` (Number) Set headroom node count. Nodes will be started in case there are no free nodes left to pick up new jobs. - `image_version` (String) The version of the Hive Virtual Warehouse image. -- `ldap_groups` (List of String) LDAP group names to be enabled for auth. +- `ldap_groups` (List of String) LDAP group names to be enabled to authenticate with. - `max_concurrent_isolated_queries` (Number) Maximum number of concurrent isolated queries. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. - `max_nodes_per_isolated_query` (Number) Maximum number of nodes per isolated query. If not provided, 0 will be applied. The 0 value means the query isolation functionality will be disabled. - `platform_jwt_auth` (Boolean) Value of ‘true’ automatically configures the Virtual Warehouse to support JWTs issued by the CDP JWT token provider. Value of ‘false’ does not enable JWT auth on the Virtual Warehouse. If this field is not specified, it defaults to ‘false’. @@ -105,7 +113,7 @@ output "jwt_token_gen_url" { - `jwt_token_gen_url` (String) URL to generate JWT tokens for the Virtual Warehouse by the CDP JWT token provider. Available if platform JWT authentication is enabled. - `kerberos_jdbc_url` (String) Kerberos JDBC URL for the Hive Virtual Warehouse. - `last_updated` (String) Timestamp of the last Terraform update of the order. -- `status` (String) The status of the database catalog. +- `status` (String) The status of the Hive Virtual Warehouse. ### Nested Schema for `aws_options` diff --git a/examples/resources/cdp_dw_vw_hive/resource.tf b/examples/resources/cdp_dw_vw_hive/resource.tf index f6d434fe..dad5463a 100644 --- a/examples/resources/cdp_dw_vw_hive/resource.tf +++ b/examples/resources/cdp_dw_vw_hive/resource.tf @@ -8,22 +8,30 @@ # OF ANY KIND, either express or implied. Refer to the License for the specific # permissions and limitations governing your use of the file. +terraform { + required_providers { + cdp = { + source = "cloudera/cdp" + } + } +} + resource "cdp_dw_vw_hive" "example" { - cluster_id = "env-id" - database_catalog_id = "warehouse-id" - name = "default-catalog" - group_size = 2 - platform_jwt_auth = true - enable_sso = true - image_version = "2024.0.18.4-5" - min_group_count = 1 - max_group_count = 3 - disable_auto_suspend = false - auto_suspend_timeout_seconds = 100 + cluster_id = "env-id" + database_catalog_id = "warehouse-id" + name = "default-catalog" + group_size = 2 + platform_jwt_auth = true + enable_sso = true + image_version = "2024.0.18.4-5" + min_group_count = 1 + max_group_count = 3 + disable_auto_suspend = false + auto_suspend_timeout_seconds = 100 scale_wait_time_seconds = 230 // either headroom or scale_wait_time_seconds can be configured - headroom = 1 + headroom = 1 max_concurrent_isolated_queries = 5 - max_nodes_per_isolated_query = 2 + max_nodes_per_isolated_query = 2 aws_options = { availability_zone = "us-west-2a" ebs_llap_spill_gb = 300 @@ -50,5 +58,5 @@ output "jwt_connection_string" { } output "jwt_token_gen_url" { - value = cdp_dw_vw_hive.example.jwt_token_gen_url + value = cdp_dw_vw_hive.example.jwt_token_gen_url } \ No newline at end of file diff --git a/resources/dw/virtualwarehouse/hive/model_hive_vw.go b/resources/dw/virtualwarehouse/hive/model_hive_vw.go index 85b98463..4518da45 100644 --- a/resources/dw/virtualwarehouse/hive/model_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/model_hive_vw.go @@ -11,9 +11,10 @@ package hive import ( - "strings" + "context" "time" + "github.com/hashicorp/terraform-plugin-framework/diag" "github.com/hashicorp/terraform-plugin-framework/types" "github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/dw/models" @@ -60,30 +61,24 @@ func (p *resourceModel) GetPollingOptions() *utils.PollingOptions { return p.PollingOptions } -func (p *resourceModel) convertToCreateVwRequest() *models.CreateVwRequest { +func (p *resourceModel) convertToCreateVwRequest(ctx context.Context) (*models.CreateVwRequest, diag.Diagnostics) { vwType := models.VwType("hive") + tags, diags := p.getTags(ctx) return &models.CreateVwRequest{ ClusterID: p.ClusterID.ValueStringPointer(), DbcID: p.DatabaseCatalogID.ValueStringPointer(), EbsLLAPSpillGB: p.getEbsLLAPSpillGB(), - ImageVersion: p.getImageVersion(), + ImageVersion: p.ImageVersion.ValueString(), Name: p.Name.ValueStringPointer(), NodeCount: utils.Int64To32(p.GroupSize), PlatformJwtAuth: p.PlatformJwtAuth.ValueBoolPointer(), QueryIsolationOptions: p.getQueryIsolationOptions(), Autoscaling: p.getAutoscaling(), AvailabilityZone: p.getAvailabilityZone(), - Tags: p.getTags(), + Tags: tags, Config: p.getServiceConfig(), VwType: &vwType, - } -} - -func (p *resourceModel) getImageVersion() string { - if p.ImageVersion.IsNull() || p.ImageVersion.IsUnknown() || p.ImageVersion.String() == "" { - return "" - } - return strings.TrimPrefix(strings.TrimSuffix(p.ImageVersion.String(), "\""), "\"") + }, diags } func (p *resourceModel) getServiceConfig() *models.ServiceConfigReq { @@ -95,22 +90,23 @@ func (p *resourceModel) getServiceConfig() *models.ServiceConfigReq { } } -func (p *resourceModel) getTags() []*models.TagRequest { +func (p *resourceModel) getTags(ctx context.Context) ([]*models.TagRequest, diag.Diagnostics) { if p.AwsOptions.Tags.IsNull() { - return nil + return nil, diag.Diagnostics{} } - var tags []*models.TagRequest - for k, v := range p.AwsOptions.Tags.Elements() { - if v.IsNull() { + tags := make([]*models.TagRequest, 0, len(p.AwsOptions.Tags.Elements())) + elements := make(map[string]string, len(p.AwsOptions.Tags.Elements())) + diags := p.AwsOptions.Tags.ElementsAs(ctx, &elements, false) + for k, v := range elements { + if v == "" { continue } - value := strings.TrimPrefix(strings.TrimSuffix(v.String(), "\""), "\"") tags = append(tags, &models.TagRequest{ Key: &k, - Value: &value, + Value: &v, }) } - return tags + return tags, diags } func (p *resourceModel) getQueryIsolationOptions() *models.QueryIsolationOptionsRequest { @@ -146,6 +142,9 @@ func (p *resourceModel) getAutoscaling() *models.AutoscalingOptionsCreateRequest } func (p *resourceModel) setFromDescribeVwResponse(resp *models.DescribeVwResponse) { + if resp.Vw == nil { + return + } p.ID = types.StringValue(resp.Vw.ID) p.DatabaseCatalogID = types.StringValue(resp.Vw.DbcID) p.Name = types.StringValue(resp.Vw.Name) diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go index e4b9d29a..f960b86d 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw.go @@ -75,8 +75,13 @@ func (r *hiveResource) Create(ctx context.Context, req resource.CreateRequest, r return } + createReq, diags := plan.convertToCreateVwRequest(ctx) + resp.Diagnostics.Append(diags...) + if resp.Diagnostics.HasError() { + return + } vw := operations.NewCreateVwParamsWithContext(ctx). - WithInput(plan.convertToCreateVwRequest()) + WithInput(createReq) response, err := r.client.Dw.Operations.CreateVw(vw) if err != nil { diff --git a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go index 18f7fd10..1bb16415 100644 --- a/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go +++ b/resources/dw/virtualwarehouse/hive/resource_hive_vw_test.go @@ -83,7 +83,7 @@ var testHiveSchema = schema.Schema{ "ldap_groups": schema.ListAttribute{ Optional: true, ElementType: types.StringType, - MarkdownDescription: "LDAP group names to be enabled for auth.", + MarkdownDescription: "LDAP group names to be enabled to authenticate with.", }, "enable_sso": schema.BoolAttribute{ Optional: true, @@ -175,7 +175,7 @@ var testHiveSchema = schema.Schema{ }, "status": schema.StringAttribute{ Computed: true, - MarkdownDescription: "The status of the database catalog.", + MarkdownDescription: "The status of the Hive Virtual Warehouse.", }, "polling_options": schema.SingleNestedAttribute{ MarkdownDescription: "Polling related configuration options that could specify various values that will be used during CDP resource creation.", @@ -582,6 +582,7 @@ func (suite *HiveTestSuite) TestStateRefresh_FailureThresholdReached() { } func (suite *HiveTestSuite) TestConvertToCreateVwRequest_All() { + ctx := context.TODO() plan := resourceModel{ ClusterID: types.StringValue("cluster-id"), DatabaseCatalogID: types.StringValue("database-catalog-id"), @@ -606,7 +607,8 @@ func (suite *HiveTestSuite) TestConvertToCreateVwRequest_All() { }, } - req := plan.convertToCreateVwRequest() + req, diags := plan.convertToCreateVwRequest(ctx) + suite.False(diags.HasError()) suite.Equal("cluster-id", *req.ClusterID) suite.Equal("database-catalog-id", *req.DbcID) suite.Equal("test-name", *req.Name) @@ -630,12 +632,14 @@ func (suite *HiveTestSuite) TestConvertToCreateVwRequest_All() { } func (suite *HiveTestSuite) TestConvertToCreateVwRequest_MissingImageVersion() { + ctx := context.TODO() plan := resourceModel{ ImageVersion: types.StringUnknown(), AwsOptions: &awsOptions{}, } - req := plan.convertToCreateVwRequest() + req, diags := plan.convertToCreateVwRequest(ctx) + suite.False(diags.HasError()) suite.Equal("", req.ImageVersion) } diff --git a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go index e9fe8d35..a8c679b9 100644 --- a/resources/dw/virtualwarehouse/hive/schema_hive_vw.go +++ b/resources/dw/virtualwarehouse/hive/schema_hive_vw.go @@ -65,7 +65,7 @@ var hiveSchema = schema.Schema{ "ldap_groups": schema.ListAttribute{ Optional: true, ElementType: types.StringType, - MarkdownDescription: "LDAP group names to be enabled for auth.", + MarkdownDescription: "LDAP group names to be enabled to authenticate with.", }, "enable_sso": schema.BoolAttribute{ Optional: true, @@ -157,7 +157,7 @@ var hiveSchema = schema.Schema{ }, "status": schema.StringAttribute{ Computed: true, - MarkdownDescription: "The status of the database catalog.", + MarkdownDescription: "The status of the Hive Virtual Warehouse.", }, "polling_options": schema.SingleNestedAttribute{ MarkdownDescription: "Polling related configuration options that could specify various values that will be used during CDP resource creation.",