⭐ terraform.resources { related } support (#2159)

Discover all related resources of a given terraform resource. Additionally overhaul the internal discovery of all terraform blocks and reduce the amount of repetition drastically.

Signed-off-by: Dominik Richter <[email protected]>

providers/terraform/resources/hcl.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/ext/typeexpr"
@@ -23,8 +24,12 @@ import (
 	"go.mondoo.com/cnquery/v9/types"
 )
 
-func (t *mqlTerraform) id() (string, error) {
-	return "terraform", nil
+type mqlTerraformInternal struct {
+	blocksByName  map[string]*mqlTerraformBlock
+	relatedBlocks map[string][]*mqlTerraformBlock
+	// these are blocks with the type set to "terraform", used for settings
+	terraformBlocks []*mqlTerraformBlock
+	lock            sync.Mutex
 }
 
 func (t *mqlTerraform) files() ([]interface{}, error) {
@@ -90,56 +95,159 @@ func (t *mqlTerraform) blocks() ([]interface{}, error) {
 		}
 		mqlHclBlocks = append(mqlHclBlocks, blocks...)
 	}
-	return mqlHclBlocks, nil
+
+	return mqlHclBlocks, t.refreshCache(mqlHclBlocks)
 }
 
-func filterBlockByType(runtime *plugin.Runtime, filterType string) ([]interface{}, error) {
-	conn := runtime.Connection.(*connection.Connection)
-	parsed := conn.Parser()
-	if parsed == nil {
-		// no results, because this is not a regular parsed HCL
-		return []interface{}{}, nil
+func (t *mqlTerraform) refreshCache(blocks []interface{}) error {
+	if blocks == nil {
+		raw := t.GetBlocks()
+		return raw.Error
 	}
 
-	files := parsed.Files()
+	t.lock.Lock()
+	if t.blocksByName != nil {
+		return nil
+	}
+	defer t.lock.Unlock()
+
+	t.blocksByName = map[string]*mqlTerraformBlock{}
+	t.relatedBlocks = map[string][]*mqlTerraformBlock{}
+
+	t.Providers.State = plugin.StateIsSet
+	t.Providers.Data = []interface{}{}
+	t.Datasources.State = plugin.StateIsSet
+	t.Datasources.Data = []interface{}{}
+	t.Resources.State = plugin.StateIsSet
+	t.Resources.Data = []interface{}{}
+	t.Variables.State = plugin.StateIsSet
+	t.Variables.Data = []interface{}{}
+	t.Outputs.State = plugin.StateIsSet
+	t.Outputs.Data = []interface{}{}
+	t.terraformBlocks = []*mqlTerraformBlock{}
+
+	for i := range blocks {
+		block := blocks[i].(*mqlTerraformBlock)
+
+		// type must be pre-initialized
+		typ := block.Type.Data
+
+		switch typ {
+		case "provider":
+			t.Providers.Data = append(t.Providers.Data, block)
+		case "data":
+			t.Datasources.Data = append(t.Providers.Data, block)
+		case "resource":
+			t.Resources.Data = append(t.Resources.Data, block)
+		case "variable":
+			t.Variables.Data = append(t.Variables.Data, block)
+		case "output":
+			t.Outputs.Data = append(t.Outputs.Data, block)
+		case "terraform":
+			t.terraformBlocks = append(t.terraformBlocks, block)
+		default:
+			// Note: we don't do anything with these blocks yet.
+			// They might be worth looking into.
+		}
 
-	var mqlHclBlocks []interface{}
-	for k := range files {
-		f := files[k]
-		blocks, err := listHclBlocks(runtime, f.Body, f)
+		// labels must be pre-initialized
+		name := block.terraformID()
+		t.blocksByName[name] = block
+	}
+
+	// We need blocks by name before we can jump into
+	// related blocks, because we need to access them
+	// via their name
+	for i := range blocks {
+		block := blocks[i].(*mqlTerraformBlock)
+		name := block.terraformID()
+
+		rel, err := listRelatedBlocks(t, block.block.Data.Body)
 		if err != nil {
-			return nil, err
+			return err
 		}
 
-		for i := range blocks {
-			b := blocks[i].(*mqlTerraformBlock)
-			blockType := b.Type.Data
-			if blockType == filterType {
-				mqlHclBlocks = append(mqlHclBlocks, b)
+		// connect from this block to all related blocks...
+		t.relatedBlocks[name] = append(t.relatedBlocks[name], rel...)
+		// ... and also connect the inverse (related to this block)
+		for i := range rel {
+			relBlock := rel[i]
+			relName := relBlock.terraformID()
+			t.relatedBlocks[relName] = append(t.relatedBlocks[relName], block)
+		}
+	}
+
+	for k, v := range t.relatedBlocks {
+		block, ok := t.blocksByName[k]
+		if !ok {
+			return errors.New("cannot find terraform block by name: " + k)
+		}
+
+		vi := make([]interface{}, len(v))
+		for i := range v {
+			vi[i] = v[i]
+		}
+
+		block.Related = plugin.TValue[[]interface{}]{
+			State: plugin.StateIsSet,
+			Data:  vi,
+		}
+	}
+
+	return nil
+}
+
+func (g *mqlTerraformBlock) terraformID() string {
+	labels := g.Labels.Data
+	var namearr []string
+	for i := range labels {
+		if s, ok := labels[i].(string); ok {
+			namearr = append(namearr, s)
+		}
+	}
+	return strings.Join(namearr, "\x00")
+}
+
+func listRelatedBlocks(t *mqlTerraform, rawBody interface{}) ([]*mqlTerraformBlock, error) {
+	var res []*mqlTerraformBlock
+	switch body := rawBody.(type) {
+	case *hclsyntax.Body:
+		for _, v := range body.Attributes {
+			refs := getReferences(v.Expr, &hcl.EvalContext{
+				Functions: hclFunctions(),
+			})
+			// we need the resource name and its ID at least
+			if len(refs) < 2 {
+				continue
 			}
+
+			refName := strings.Join(refs[0:2], "\x00")
+			res = append(res, t.blocksByName[refName])
 		}
+	case hcl.Body:
+		return nil, errors.New("cannot yet list related blocks for regular hcl Body")
 	}
-	return mqlHclBlocks, nil
+	return res, nil
 }
 
 func (t *mqlTerraform) providers() ([]interface{}, error) {
-	return filterBlockByType(t.MqlRuntime, "provider")
+	return nil, t.refreshCache(nil)
 }
 
 func (t *mqlTerraform) datasources() ([]interface{}, error) {
-	return filterBlockByType(t.MqlRuntime, "data")
+	return nil, t.refreshCache(nil)
 }
 
 func (t *mqlTerraform) resources() ([]interface{}, error) {
-	return filterBlockByType(t.MqlRuntime, "resource")
+	return nil, t.refreshCache(nil)
 }
 
 func (t *mqlTerraform) variables() ([]interface{}, error) {
-	return filterBlockByType(t.MqlRuntime, "variable")
+	return nil, t.refreshCache(nil)
 }
 
 func (t *mqlTerraform) outputs() ([]interface{}, error) {
-	return filterBlockByType(t.MqlRuntime, "output")
+	return nil, t.refreshCache(nil)
 }
 
 func extractHclCodeSnippet(file *hcl.File, fileRange hcl.Range) string {
@@ -399,6 +507,28 @@ func getCtyValue(expr hcl.Expression, ctx *hcl.EvalContext) interface{} {
 	}
 }
 
+func getReferences(expr hcl.Expression, ctx *hcl.EvalContext) []string {
+	switch t := expr.(type) {
+	case *hclsyntax.ScopeTraversalExpr:
+		traversal := t.Variables()
+		res := []string{}
+		for i := range traversal {
+			tr := traversal[i]
+			for j := range tr {
+				switch v := tr[j].(type) {
+				case hcl.TraverseRoot:
+					res = append(res, v.Name)
+				case hcl.TraverseAttr:
+					res = append(res, v.Name)
+				}
+			}
+		}
+		return res
+	default:
+		return nil
+	}
+}
+
 func GetKeyString(key interface{}) string {
 	switch v := key.(type) {
 	case []string:
@@ -465,6 +595,19 @@ func listHclBlocks(runtime *plugin.Runtime, rawBody interface{}, file *hcl.File)
 	return mqlHclBlocks, nil
 }
 
+func (g *mqlTerraformBlock) related() ([]interface{}, error) {
+	// This field should be default be set by the Terraform routine that
+	// initializes all blocks. If we land here from a recording or other
+	// path, re-run it.
+	o, err := CreateResource(g.MqlRuntime, "terraform", map[string]*llx.RawData{})
+	if err != nil {
+		return nil, err
+	}
+
+	terraform := o.(*mqlTerraform)
+	return nil, terraform.refreshCache(nil)
+}
+
 func newFilePosRange(runtime *plugin.Runtime, r hcl.Range) (plugin.Resource, plugin.Resource, error) {
 	start, err := CreateResource(runtime, "terraform.fileposition", map[string]*llx.RawData{
 		"path":   llx.StringData(r.Filename),
@@ -545,11 +688,17 @@ func (t *mqlTerraformModule) block() (*mqlTerraformBlock, error) {
 }
 
 func initTerraformSettings(runtime *plugin.Runtime, args map[string]*llx.RawData) (map[string]*llx.RawData, plugin.Resource, error) {
-	blocks, err := filterBlockByType(runtime, "terraform")
+	o, err := CreateResource(runtime, "terraform", map[string]*llx.RawData{})
 	if err != nil {
 		return nil, nil, err
 	}
 
+	terraform := o.(*mqlTerraform)
+	if err := terraform.refreshCache(nil); err != nil {
+		return nil, nil, err
+	}
+
+	blocks := terraform.terraformBlocks
 	if len(blocks) != 1 {
 		// no terraform settings block found, this is ok for terraform and not an error
 		// TODO: return modified arguments to load from recording
@@ -560,7 +709,7 @@ func initTerraformSettings(runtime *plugin.Runtime, args map[string]*llx.RawData
 		}, nil
 	}
 
-	settingsBlock := blocks[0].(*mqlTerraformBlock)
+	settingsBlock := blocks[0]
 	args["block"] = llx.ResourceData(settingsBlock, "terraform.block")
 	args["requiredProviders"] = llx.DictData(map[string]interface{}{})
 	args["backend"] = llx.DictData(map[string]interface{}{})

providers/terraform/resources/terraform.lr

@@ -63,6 +63,8 @@ terraform.block @defaults("type labels") {
   attributes() dict
   // Child Blocks
   blocks() []terraform.block
+  // Related blocks
+  related() []terraform.block
   // Block Snippet
   snippet string
 }

providers/terraform/resources/terraform.lr.manifest.yaml

@@ -33,6 +33,8 @@ resources:
       end: {}
       labels: {}
       nameLabel: {}
+      related:
+        min_mondoo_version: 9.0.12
       snippet: {}
       start: {}
       type: {}