Skip to content

Commit

Permalink
OmitType and TypeName to support sharing types (#2409)
Browse files Browse the repository at this point in the history 
Introduce OmitType and TypeName flags to enforce type sharing.

Some of the upstream providers generate very large concrete schemata. TF
is not being materially affected, just high RAM demands for in-memory
processing. The example is inspired by QuickSight types in AWS. Pulumi
is affected significantly. In Pulumi the default projection is going to
generate named types for every instance of the shared schema. This leads
to SDK bloat and issues with "filename too long."

With this change it is possible for the provider maintainer opt into
explicit sharing of types, and ensure that the type names for the shared
types have shorter meaningful prefixes.

At definition type the user can specify the type name to generate, which
can be very short, and replace the automatically implied
ReallyLongPrefixedTypeName like this:

```go
"visuals": {
	Elem: &info.Schema{
		TypeName: tfbridge.Ref("Visual"),
	},
},
```

At reference time in another resource, the user can reuse an already
generated type by token. This already worked before this change but had
the downside of still generating unused helper types and causing SDK
bloat.

```go
"visuals": {
	Elem: &info.Schema{
		Type:     "testprov:index/Visual:Visual",
	},
},
```

With this change it is possible to instruct the bridge to stop
generating the unused helper types:

```go
"visuals": {
	Elem: &info.Schema{
		Type:     "testprov:index/Visual:Visual",
		OmitType: true
	},
},
```
  • Loading branch information
@t0yv0
t0yv0 authored Sep 12, 2024
1 parent fbc2abd commit f0fee4f
Show file tree
Hide file tree
Showing 6 changed files with 403 additions and 3 deletions.
9 changes: 8 additions & 1 deletion pkg/tfbridge/info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,11 +318,18 @@ func MakeResource(pkg string, mod string, res string) tokens.Type {
return tokens.NewTypeToken(modT, tokens.TypeName(res))
}

// BoolRef returns a reference to the bool argument.
// BoolRef returns a reference to the bool argument. Retained for backwards compatibility. Prefer [Ref] for new usage
// and other types like strings.
func BoolRef(b bool) *bool {
return &b
}

// Fluently construct a reference to the argument. This utility function is needed to ease configuring the bridge where
// references are expected instead of plain boolean or string literals.
func Ref[T any](x T) *T {
return &x
}

// StringValue gets a string value from a property map if present, else ""
func StringValue(vars resource.PropertyMap, prop resource.PropertyKey) string {
val, ok := vars[prop]
Expand Down
30 changes: 29 additions & 1 deletion pkg/tfbridge/info/info.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,9 +441,19 @@ type Schema struct {
// a name to override the default when targeting C#; "" uses the default.
CSharpName string

// a type to override the default; "" uses the default.
// An optional Pulumi type token to use for the Pulumi type projection of the current property. When unset, the
// default behavior is to generate fresh named Pulumi types as needed to represent the schema. To force the use
// of a known type and avoid generating unnecessary types, use both [Type] and [OmitType].
Type tokens.Type

// Used together with [Type] to omit generating any Pulumi types whatsoever for the current property, and
// instead use the object type identified by the token setup in [Type].
//
// It is an error to set [OmitType] to true without specifying [Type].
//
// Experimental.
OmitType bool

// alternative types that can be used instead of the override.
AltTypes []tokens.Type

Expand Down Expand Up @@ -502,6 +512,24 @@ type Schema struct {

// whether or not to treat this property as secret
Secret *bool

// Specifies the exact name to use for the generated type.
//
// When generating types for properties, by default Pulumi picks reasonable names based on the property path
// prefix and the name of the property. Use [TypeName] to override this decision when the default names for
// nested properties are too long or otherwise undesirable. The choice will further affect the automatically
// generated names for any properties nested under the current one.
//
// Example use:
//
// TypeName: tfbridge.Ref("Visual")
//
// Note that the type name, and not the full token like "aws:quicksight/Visual:Visual" is specified. The token
// will be picked based on the current module ("quicksight" in the above example) where the parent resource or
// data source is found.
//
// Experimental.
TypeName *string
}

// Config represents a synthetic configuration variable that is Pulumi-only, and not passed to Terraform.
Expand Down
27 changes: 27 additions & 0 deletions pkg/tfgen/generate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,8 @@ type propertyType struct {
nestedType tokens.Type
altTypes []tokens.Type
asset *tfbridge.AssetTranslation

typeName *string
}

func (g *Generator) Sink() diag.Sink {
Expand All @@ -353,6 +355,9 @@ func (g *Generator) makePropertyType(typePath paths.TypePath,
entityDocs entityDocs) (*propertyType, error) {

t := &propertyType{}
if info != nil {
t.typeName = info.TypeName
}

var elemInfo *tfbridge.SchemaInfo
if info != nil {
Expand Down Expand Up @@ -456,10 +461,23 @@ func getDocsFromSchemaMap(key string, schemaMap shim.SchemaMap) string {
func (g *Generator) makeObjectPropertyType(typePath paths.TypePath,
res shim.Resource, info *tfbridge.SchemaInfo,
out bool, entityDocs entityDocs) (*propertyType, error) {

// If the user supplied an explicit Type token override, omit generating types and short-circuit.
if info != nil && info.OmitType {
if info.Type == "" {
return nil, fmt.Errorf("Cannot set info.OmitType without also setting info.Type")
}
return &propertyType{typ: info.Type}, nil
}

t := &propertyType{
kind: kindObject,
}

if info != nil {
t.typeName = info.TypeName
}

if info != nil {
t.typ = info.Type
t.nestedType = info.NestedType
Expand Down Expand Up @@ -549,6 +567,15 @@ func (t *propertyType) equals(other *propertyType) bool {
if len(t.properties) != len(other.properties) {
return false
}
switch {
case t.typeName != nil && other.typeName == nil:
return false
case t.typeName == nil && other.typeName != nil:
return false
case t.typeName != nil && other.typeName != nil &&
*t.typeName != *other.typeName:
return false
}
for i, p := range t.properties {
o := other.properties[i]
if p.name != o.name {
Expand Down
10 changes: 9 additions & 1 deletion pkg/tfgen/generate_schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,15 @@ func (nt *schemaNestedTypes) declareType(typePath paths.TypePath, declarer decla
typ *propertyType, isInput bool) string {

// Generate a name for this nested type.
typeName := namePrefix + cases.Title(language.Und, cases.NoLower).String(name)
var typeName string

if typ.typeName != nil {
// Use an explicit name if provided.
typeName = *typ.typeName
} else {
// Otherwise build one based on the current property name and prefix.
typeName = namePrefix + cases.Title(language.Und, cases.NoLower).String(name)
}

// Override the nested type name, if necessary.
if typ.nestedType.Name().String() != "" {
Expand Down
166 changes: 166 additions & 0 deletions pkg/tfgen/generate_schema_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@ import (
"encoding/json"
"fmt"
"io"
"runtime"
"sort"
"testing"
"text/template"

"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
"github.com/hexops/autogold/v2"
csgen "github.com/pulumi/pulumi/pkg/v3/codegen/dotnet"
gogen "github.com/pulumi/pulumi/pkg/v3/codegen/go"
Expand All @@ -35,7 +38,9 @@ import (

bridgetesting "github.com/pulumi/pulumi-terraform-bridge/v3/internal/testing"
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge"
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfbridge/info"
"github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfgen/internal/testprovider"
sdkv2 "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tfshim/sdk-v2"
"github.com/pulumi/pulumi-terraform-bridge/v3/unstable/metadata"
"github.com/pulumi/pulumi-terraform-bridge/x/muxer"
)
Expand Down Expand Up @@ -116,6 +121,167 @@ func TestCSharpMiniRandom(t *testing.T) {
bridgetesting.AssertEqualsJSONFile(t, "test_data/minirandom-schema-csharp.json", schema)
}

// Test the ability to force type sharing. Some of the upstream providers generate very large concrete schemata in Go,
// with TF not being materially affected. The example is inspired by QuickSight types in AWS. In Pulumi the default
// projection is going to generate named types for every instance of the shared schema. This may lead to SDK bloat. Test
// the ability of the provider author to curb the bloat and force an explicit sharing.
func TestTypeSharing(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skipf("Skipping on Windows due to a test setup issue")
}

tmpdir := t.TempDir()
barCharVisualSchema := func() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Optional: true,
MinItems: 1,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"nest": {
Type: schema.TypeList,
MaxItems: 1,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"nested_prop": {
Type: schema.TypeBool,
Optional: true,
},
},
},
},
},
},
}
}
visualsSchema := func() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
MinItems: 1,
MaxItems: 50,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"bar_chart_visual": barCharVisualSchema(),
"box_plot_visual": barCharVisualSchema(),
},
},
}
}
provider := info.Provider{
Name: "testprov",
P: sdkv2.NewProvider(&schema.Provider{
ResourcesMap: map[string]*schema.Resource{
"testprov_r1": {
Schema: map[string]*schema.Schema{
"sheets": {
Type: schema.TypeList,
MinItems: 1,
MaxItems: 20,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"visuals": visualsSchema(),
},
},
},
},
},
"testprov_r2": {
Schema: map[string]*schema.Schema{
"x": {
Type: schema.TypeInt,
Optional: true,
},
"sheets": {
Type: schema.TypeList,
MinItems: 1,
MaxItems: 20,
Optional: true,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"y": {
Type: schema.TypeBool,
Optional: true,
},
"visuals": visualsSchema(),
},
},
},
},
},
},
}),
UpstreamRepoPath: tmpdir,
Resources: map[string]*info.Resource{
"testprov_r1": {
Tok: "testprov:index:R1",
Fields: map[string]*info.Schema{
"sheets": {
Elem: &info.Schema{
Fields: map[string]*info.Schema{
"visuals": {
Elem: &info.Schema{
TypeName: tfbridge.Ref("Visual"),
},
},
},
},
},
},
},
"testprov_r2": {
Tok: "testprov:index:R2",
Fields: map[string]*info.Schema{
"sheets": {
Elem: &info.Schema{
Fields: map[string]*info.Schema{
"visuals": {
Elem: &info.Schema{
Type: "testprov:index/Visual:Visual",
OmitType: true,
},
},
},
},
},
},
},
},
}

var buf bytes.Buffer
schema, err := GenerateSchema(provider, diag.DefaultSink(&buf, &buf, diag.FormatOptions{
Color: colors.Never,
}))
require.NoError(t, err)

t.Logf("%s", buf.String())

keys := []string{}
for k := range schema.Types {
keys = append(keys, k)
}
sort.Strings(keys)

// Note that there is only one set of helper types, and they are not prefixed by any of the resource names.
autogold.Expect([]string{
"testprov:index/R1Sheet:R1Sheet", "testprov:index/R2Sheet:R2Sheet",
"testprov:index/Visual:Visual",
"testprov:index/VisualBarChartVisual:VisualBarChartVisual",
"testprov:index/VisualBarChartVisualNest:VisualBarChartVisualNest",
"testprov:index/VisualBoxPlotVisual:VisualBoxPlotVisual",
"testprov:index/VisualBoxPlotVisualNest:VisualBoxPlotVisualNest",
}).Equal(t, keys)

bytes, err := json.MarshalIndent(schema, "", " ")
require.NoError(t, err)

autogold.ExpectFile(t, autogold.Raw(string(bytes)))
}

// TestPropertyDocumentationEdits tests that documentation edits are applied to
// individual properties. This includes both the property description and
// deprecation message. This tests the following workflow
Expand Down
Loading

0 comments on commit f0fee4f

Please sign in to comment.