Skip to content

Commit

Permalink
ENGESC-23979 - Unable to create Data lake and Data hub cluster throug…
Browse files Browse the repository at this point in the history 
…h Terraform
  • Loading branch information
@gregito
gregito committed Feb 26, 2024
1 parent 73ccef9 commit 4b54cd2
Show file tree
Hide file tree
Showing 20 changed files with 203 additions and 124 deletions.
12 changes: 0 additions & 12 deletions resources/datahub/common_scheme.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,18 +57,6 @@ var generalAttributes = map[string]schema.Attribute{
MarkdownDescription: "The name of the cluster.",
Required: true,
},
"cluster_template": schema.StringAttribute{
MarkdownDescription: "The name of the cluster template.",
Required: true,
},
"cluster_definition": schema.StringAttribute{
MarkdownDescription: "The name of the cluster definition.",
Required: true,
},
"environment": schema.StringAttribute{
MarkdownDescription: "The name of the environment where the cluster will belong to.",
Required: true,
},
"destroy_options": schema.SingleNestedAttribute{
Optional: true,
Description: "Cluster deletion options.",
Expand Down
18 changes: 0 additions & 18 deletions resources/datahub/common_scheme_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,24 +54,6 @@ var commonElementCaseSet = []TestCaseStructure{
computed: false,
shouldBeRequired: true,
},
{
name: "cluster_template must exist",
field: "cluster_template",
computed: false,
shouldBeRequired: true,
},
{
name: "cluster_definition must exist",
field: "cluster_definition",
computed: false,
shouldBeRequired: true,
},
{
name: "environment must exist",
field: "environment",
computed: false,
shouldBeRequired: true,
},
{
name: "destroy_options should exist",
field: "destroy_options",
Expand Down
44 changes: 19 additions & 25 deletions resources/datahub/converter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,21 @@ import (
datahubmodels "github.com/cloudera/terraform-provider-cdp/cdp-sdk-go/gen/datahub/models"
)

func fromModelToAwsRequest(model datahubResourceModel, ctx context.Context) *datahubmodels.CreateAWSClusterRequest {
debug(ctx, "Conversion from datahubResourceModel to CreateAWSClusterRequest started.")
func fromModelToAwsRequest(model awsDatahubResourceModel, ctx context.Context) *datahubmodels.CreateAWSClusterRequest {
tflog.Debug(ctx, "Conversion from datahubResourceModel to CreateAWSClusterRequest started.")
req := datahubmodels.CreateAWSClusterRequest{}
req.ClusterName = model.Name.ValueString()
req.ClusterTemplate = model.ClusterTemplate.ValueString()
req.Environment = model.Environment.ValueString()
req.ClusterDefinition = model.ClusterDefinition.ValueString()
var igs []*datahubmodels.InstanceGroupRequest
debug(ctx, fmt.Sprintf("%d instance group found in the input model.", len(model.InstanceGroup)))
tflog.Debug(ctx, fmt.Sprintf("%d instance group found in the input model.", len(model.InstanceGroup)))
for _, group := range model.InstanceGroup {
debug(ctx, fmt.Sprintf("Converting InstanceGroupRequest: %+v.", group))
tflog.Debug(ctx, fmt.Sprintf("Converting InstanceGroupRequest: %+v.", group))
var volReqs []*datahubmodels.AttachedVolumeRequest
debug(ctx, fmt.Sprintf("%d attached volume request found in the input model.", len(model.InstanceGroup)))
tflog.Debug(ctx, fmt.Sprintf("%d attached volume request found in the input model.", len(model.InstanceGroup)))
for _, vrs := range group.AttachedVolumeConfiguration {
debug(ctx, fmt.Sprintf("Converting AttachedVolumeConfiguration: %+v.", vrs))
tflog.Debug(ctx, fmt.Sprintf("Converting AttachedVolumeConfiguration: %+v.", vrs))
volReqs = append(volReqs, createAttachedVolumeRequest(vrs))
}
var igRecipes []string
Expand All @@ -58,25 +58,25 @@ func fromModelToAwsRequest(model datahubResourceModel, ctx context.Context) *dat
igs = append(igs, ig)
}
req.InstanceGroups = igs
debug(ctx, "Conversion from datahubResourceModel to CreateAWSClusterRequest has finished.")
tflog.Debug(ctx, fmt.Sprintf("Conversion from datahubResourceModel to CreateAWSClusterRequest has finished with request: %+v.", req))
return &req
}

func fromModelToGcpRequest(model gcpDatahubResourceModel, ctx context.Context) *datahubmodels.CreateGCPClusterRequest {
debug(ctx, "Conversion from gcpDatahubResourceModel to CreateGCPClusterRequest started.")
tflog.Debug(ctx, "Conversion from gcpDatahubResourceModel to CreateGCPClusterRequest started.")
req := datahubmodels.CreateGCPClusterRequest{}
req.ClusterName = model.Name.ValueString()
req.EnvironmentName = model.Environment.ValueString()
req.ClusterTemplateName = model.ClusterTemplate.ValueString()
req.ClusterDefinitionName = model.ClusterDefinition.ValueString()
var igs []*datahubmodels.GCPInstanceGroupRequest
debug(ctx, fmt.Sprintf("%d instance group found in the input model.", len(model.InstanceGroup)))
tflog.Debug(ctx, fmt.Sprintf("%d instance group found in the input model.", len(model.InstanceGroup)))
for _, group := range model.InstanceGroup {
debug(ctx, fmt.Sprintf("Converting GCPInstanceGroupRequest: %+v.", group))
tflog.Debug(ctx, fmt.Sprintf("Converting GCPInstanceGroupRequest: %+v.", group))
var volReqs []*datahubmodels.AttachedVolumeRequest
debug(ctx, fmt.Sprintf("%d attached volume request found in the input model.", len(model.InstanceGroup)))
tflog.Debug(ctx, fmt.Sprintf("%d attached volume request found in the input model.", len(model.InstanceGroup)))
for _, vrs := range group.AttachedVolumeConfiguration {
debug(ctx, fmt.Sprintf("Converting AttachedVolumeConfiguration: %+v.", vrs))
tflog.Debug(ctx, fmt.Sprintf("Converting AttachedVolumeConfiguration: %+v.", vrs))
volReqs = append(volReqs, createAttachedVolumeRequest(vrs))
}
var igRecipes []string
Expand All @@ -99,26 +99,26 @@ func fromModelToGcpRequest(model gcpDatahubResourceModel, ctx context.Context) *
igs = append(igs, ig)
}
req.InstanceGroups = igs
debug(ctx, "Conversion from gcpDatahubResourceModel to CreateGCPClusterRequest has finished.")
tflog.Debug(ctx, "Conversion from gcpDatahubResourceModel to CreateGCPClusterRequest has finished.")
return &req
}

func fromModelToAzureRequest(model datahubResourceModel, ctx context.Context) *datahubmodels.CreateAzureClusterRequest {
debug(ctx, "Conversion from datahubResourceModel to CreateAzureClusterRequest started.")
func fromModelToAzureRequest(model azureDatahubResourceModel, ctx context.Context) *datahubmodels.CreateAzureClusterRequest {
tflog.Debug(ctx, "Conversion from datahubResourceModel to CreateAzureClusterRequest started.")
req := datahubmodels.CreateAzureClusterRequest{}
req.DatabaseType = model.DatabaseType.ValueString()
req.ClusterName = model.Name.ValueString()
req.ClusterTemplateName = model.ClusterTemplate.ValueString()
req.EnvironmentName = model.Environment.ValueString()
req.ClusterDefinitionName = model.ClusterDefinition.ValueString()
var igs []*datahubmodels.AzureInstanceGroupRequest
debug(ctx, fmt.Sprintf("%d instance group found in the input model.", len(model.InstanceGroup)))
tflog.Debug(ctx, fmt.Sprintf("%d instance group found in the input model.", len(model.InstanceGroup)))
for _, group := range model.InstanceGroup {
debug(ctx, fmt.Sprintf("Converting InstanceGroupRequest: %+v.", group))
tflog.Debug(ctx, fmt.Sprintf("Converting InstanceGroupRequest: %+v.", group))
var volReqs []*datahubmodels.AttachedVolumeRequest
debug(ctx, fmt.Sprintf("%d attached volume request found in the input model.", len(model.InstanceGroup)))
tflog.Debug(ctx, fmt.Sprintf("%d attached volume request found in the input model.", len(model.InstanceGroup)))
for _, vrs := range group.AttachedVolumeConfiguration {
debug(ctx, fmt.Sprintf("Converting AttachedVolumeConfiguration: %+v.", vrs))
tflog.Debug(ctx, fmt.Sprintf("Converting AttachedVolumeConfiguration: %+v.", vrs))
volReqs = append(volReqs, createAttachedVolumeRequest(vrs))
}
var igRecipes []string
Expand Down Expand Up @@ -162,9 +162,3 @@ func int64To32Pointer(in types.Int64) *int32 {
var n2 = int32(n64)
return &n2
}

func debug(ctx context.Context, msg string) {
if ctx != nil {
tflog.Debug(ctx, msg)
}
}
55 changes: 28 additions & 27 deletions resources/datahub/converter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,18 @@
package datahub

import (
"context"
"github.com/hashicorp/terraform-plugin-framework/types"
"testing"
)

func TestFromModelToRequestBasicFields(t *testing.T) {
input := datahubResourceModel{
input := awsDatahubResourceModel{
Name: types.StringValue("someClusterName"),
Environment: types.StringValue("someEnvironment"),
ClusterTemplate: types.StringValue("someClusterTemplateNameOrCRN"),
}
got := fromModelToAwsRequest(input, nil)
got := fromModelToAwsRequest(input, context.TODO())

compareStrings(got.ClusterName, input.Name.ValueString(), t)
compareStrings(got.Environment, input.Environment.ValueString(), t)
Expand All @@ -31,9 +32,9 @@ func TestFromModelToRequestBasicFields(t *testing.T) {
func TestFromModelToRequestRecipe(t *testing.T) {
recipes := []types.String{types.StringValue("recipe1"), types.StringValue("recipe2")}
igs := []InstanceGroup{{Recipes: recipes}}
input := datahubResourceModel{InstanceGroup: igs}
input := awsDatahubResourceModel{InstanceGroup: igs}

got := fromModelToAwsRequest(input, nil)
got := fromModelToAwsRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(input.InstanceGroup), t)
compareInts(len(got.InstanceGroups[0].RecipeNames), len(input.InstanceGroup[0].Recipes), t)
Expand All @@ -58,9 +59,9 @@ func TestFromModelToRequestAttachedVolumeConfiguration(t *testing.T) {
VolumeType: types.StringValue("ephemeral"),
}}
igs := []InstanceGroup{{AttachedVolumeConfiguration: avcs}}
input := datahubResourceModel{InstanceGroup: igs}
input := awsDatahubResourceModel{InstanceGroup: igs}

got := fromModelToAwsRequest(input, nil)
got := fromModelToAwsRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(input.InstanceGroup), t)
compareInts(len(got.InstanceGroups[0].AttachedVolumeConfiguration), len(avcs), t)
Expand All @@ -81,9 +82,9 @@ func TestFromModelToRequestInstanceGroups(t *testing.T) {
RecoveryMode: types.StringValue("MANUAL"),
}}

input := datahubResourceModel{InstanceGroup: igs}
input := awsDatahubResourceModel{InstanceGroup: igs}

got := fromModelToAwsRequest(input, nil)
got := fromModelToAwsRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(igs), t)
resultIg := got.InstanceGroups[0]
Expand All @@ -99,9 +100,9 @@ func TestFromModelToRequestVolumeEncryption(t *testing.T) {
VolumeEncryption: VolumeEncryption{Encryption: types.BoolValue(true)},
}}

input := datahubResourceModel{InstanceGroup: igs}
input := awsDatahubResourceModel{InstanceGroup: igs}

got := fromModelToAwsRequest(input, nil)
got := fromModelToAwsRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(igs), t)
resultVolumeEncryption := got.InstanceGroups[0].VolumeEncryption
Expand All @@ -113,8 +114,8 @@ func TestFromModelToRequestVolumeEncryption(t *testing.T) {
}

func TestFromModelToRequestClusterDefinition(t *testing.T) {
input := datahubResourceModel{ClusterDefinition: types.StringValue("SomeClusterDef")}
got := fromModelToAwsRequest(input, nil)
input := awsDatahubResourceModel{ClusterDefinition: types.StringValue("SomeClusterDef")}
got := fromModelToAwsRequest(input, context.TODO())

compareStrings(got.ClusterDefinition, input.ClusterDefinition.ValueString(), t)
}
Expand All @@ -125,7 +126,7 @@ func TestFromModelToGcpRequestBasicFields(t *testing.T) {
Environment: types.StringValue("someEnvironment"),
ClusterTemplate: types.StringValue("someClusterTemplateNameOrCRN"),
}
got := fromModelToGcpRequest(input, nil)
got := fromModelToGcpRequest(input, context.TODO())

compareStrings(got.ClusterName, input.Name.ValueString(), t)
compareStrings(got.EnvironmentName, input.Environment.ValueString(), t)
Expand All @@ -137,7 +138,7 @@ func TestFromModelToGcpRequestRecipe(t *testing.T) {
igs := []GcpInstanceGroup{{Recipes: recipes}}
input := gcpDatahubResourceModel{InstanceGroup: igs}

got := fromModelToGcpRequest(input, nil)
got := fromModelToGcpRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(input.InstanceGroup), t)
compareInts(len(got.InstanceGroups[0].RecipeNames), len(input.InstanceGroup[0].Recipes), t)
Expand All @@ -164,7 +165,7 @@ func TestFromModelToGcpRequestAttachedVolumeConfiguration(t *testing.T) {
igs := []GcpInstanceGroup{{AttachedVolumeConfiguration: avcs}}
input := gcpDatahubResourceModel{InstanceGroup: igs}

got := fromModelToGcpRequest(input, nil)
got := fromModelToGcpRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(input.InstanceGroup), t)
compareInts(len(got.InstanceGroups[0].AttachedVolumeConfiguration), len(avcs), t)
Expand All @@ -187,7 +188,7 @@ func TestFromModelToGcpRequestInstanceGroups(t *testing.T) {

input := gcpDatahubResourceModel{InstanceGroup: igs}

got := fromModelToGcpRequest(input, nil)
got := fromModelToGcpRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(igs), t)
resultIg := got.InstanceGroups[0]
Expand All @@ -200,18 +201,18 @@ func TestFromModelToGcpRequestInstanceGroups(t *testing.T) {

func TestFromModelToGcpRequestClusterDefinition(t *testing.T) {
input := gcpDatahubResourceModel{ClusterDefinition: types.StringValue("SomeClusterDef")}
got := fromModelToGcpRequest(input, nil)
got := fromModelToGcpRequest(input, context.TODO())

compareStrings(got.ClusterDefinitionName, input.ClusterDefinition.ValueString(), t)
}

func TestFromModelToAzureRequestBasicFields(t *testing.T) {
input := datahubResourceModel{
input := azureDatahubResourceModel{
Name: types.StringValue("someClusterName"),
Environment: types.StringValue("someEnvironment"),
ClusterTemplate: types.StringValue("someClusterTemplateNameOrCRN"),
}
got := fromModelToAzureRequest(input, nil)
got := fromModelToAzureRequest(input, context.TODO())

compareStrings(got.ClusterName, input.Name.ValueString(), t)
compareStrings(got.EnvironmentName, input.Environment.ValueString(), t)
Expand All @@ -221,9 +222,9 @@ func TestFromModelToAzureRequestBasicFields(t *testing.T) {
func TestFromModelToAzureRequestRecipe(t *testing.T) {
recipes := []types.String{types.StringValue("recipe1"), types.StringValue("recipe2")}
igs := []InstanceGroup{{Recipes: recipes}}
input := datahubResourceModel{InstanceGroup: igs}
input := azureDatahubResourceModel{InstanceGroup: igs}

got := fromModelToAzureRequest(input, nil)
got := fromModelToAzureRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(input.InstanceGroup), t)
compareInts(len(got.InstanceGroups[0].RecipeNames), len(input.InstanceGroup[0].Recipes), t)
Expand All @@ -248,9 +249,9 @@ func TestFromModelToAzureRequestAttachedVolumeConfiguration(t *testing.T) {
VolumeType: types.StringValue("ephemeral"),
}}
igs := []InstanceGroup{{AttachedVolumeConfiguration: avcs}}
input := datahubResourceModel{InstanceGroup: igs}
input := azureDatahubResourceModel{InstanceGroup: igs}

got := fromModelToAzureRequest(input, nil)
got := fromModelToAzureRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(input.InstanceGroup), t)
compareInts(len(got.InstanceGroups[0].AttachedVolumeConfiguration), len(avcs), t)
Expand All @@ -271,9 +272,9 @@ func TestFromModelToAzureRequestInstanceGroups(t *testing.T) {
RecoveryMode: types.StringValue("MANUAL"),
}}

input := datahubResourceModel{InstanceGroup: igs}
input := azureDatahubResourceModel{InstanceGroup: igs}

got := fromModelToAzureRequest(input, nil)
got := fromModelToAzureRequest(input, context.TODO())

compareInts(len(got.InstanceGroups), len(igs), t)
resultIg := got.InstanceGroups[0]
Expand All @@ -285,8 +286,8 @@ func TestFromModelToAzureRequestInstanceGroups(t *testing.T) {
}

func TestFromModelToAzureRequestClusterDefinition(t *testing.T) {
input := datahubResourceModel{ClusterDefinition: types.StringValue("SomeClusterDef")}
got := fromModelToAzureRequest(input, nil)
input := azureDatahubResourceModel{ClusterDefinition: types.StringValue("SomeClusterDef")}
got := fromModelToAzureRequest(input, context.TODO())

compareStrings(got.ClusterDefinitionName, input.ClusterDefinition.ValueString(), t)
}
Expand Down
34 changes: 34 additions & 0 deletions resources/datahub/model_aws_datahub.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Copyright 2023 Cloudera. All Rights Reserved.
//
// This file is licensed under the Apache License Version 2.0 (the "License").
// You may not use this file except in compliance with the License.
// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0.
//
// This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
// OF ANY KIND, either express or implied. Refer to the License for the specific
// permissions and limitations governing your use of the file.

package datahub

import (
"github.com/hashicorp/terraform-plugin-framework/types"

"github.com/cloudera/terraform-provider-cdp/utils"
)

type awsDatahubResourceModel struct {
ID types.String `tfsdk:"id"`
Crn types.String `tfsdk:"crn"`
Name types.String `tfsdk:"name"`
Status types.String `tfsdk:"status"`
InstanceGroup []InstanceGroup `tfsdk:"instance_group"`
PollingOptions *utils.PollingOptions `tfsdk:"polling_options"`
DestroyOptions *DestroyOptions `tfsdk:"destroy_options"`
ClusterDefinition types.String `tfsdk:"cluster_definition"`
ClusterTemplate types.String `tfsdk:"cluster_template"`
Environment types.String `tfsdk:"environment"`
}

func (d *awsDatahubResourceModel) forceDeleteRequested() bool {
return d.DestroyOptions != nil && !d.DestroyOptions.ForceDeleteCluster.IsNull() && d.DestroyOptions.ForceDeleteCluster.ValueBool()
}
Loading

0 comments on commit 4b54cd2

Please sign in to comment.