🧹 use 1 global resource schema instead of per runtime (#3326)

* use uber mockgen instead of golang mockgen Signed-off-by: Ivan Milchev <[email protected]> * fix go.mod formatting Signed-off-by: Ivan Milchev <[email protected]> * 🧹 use 1 global resource schema instead of per runtime Signed-off-by: Ivan Milchev <[email protected]> * fix tests Signed-off-by: Ivan Milchev <[email protected]> * fix more tests Signed-off-by: Ivan Milchev <[email protected]> * make sure resource overriding and extension works Signed-off-by: Ivan Milchev <[email protected]> * fix tests Signed-off-by: Ivan Milchev <[email protected]> * fix more failing tests Signed-off-by: Ivan Milchev <[email protected]> * code cleanup Signed-off-by: Ivan Milchev <[email protected]> * more test fixes Signed-off-by: Ivan Milchev <[email protected]> * more fixes Signed-off-by: Ivan Milchev <[email protected]> * fix extensible schema test Signed-off-by: Ivan Milchev <[email protected]> * add more safe guards for merging resources Signed-off-by: Ivan Milchev <[email protected]> * make extensible schema test deterministic Signed-off-by: Ivan Milchev <[email protected]> * fix comments Signed-off-by: Ivan Milchev <[email protected]> * fix linter Signed-off-by: Ivan Milchev <[email protected]> --------- Signed-off-by: Ivan Milchev <[email protected]>
mondoohq · Feb 19, 2024 · e4b3be2 · e4b3be2
1 parent 9b54c9d
commit e4b3be2
Show file tree

Hide file tree

Showing 23 changed files with 270 additions and 165 deletions.
diff --git a/cli/printer/printer.go b/cli/printer/printer.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/muesli/termenv"
 	"go.mondoo.com/cnquery/v10/cli/theme/colors"
-	"go.mondoo.com/cnquery/v10/llx"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 )
 
 // Printer turns code into human-readable strings
@@ -23,7 +23,7 @@ type Printer struct {
 	Disabled  func(...interface{}) string
 	Failed    func(...interface{}) string
 	Success   func(...interface{}) string
-	schema    llx.Schema
+	schema    resources.ResourcesSchema
 }
 
 // DefaultPrinter that can be used without additional configuration
@@ -54,7 +54,7 @@ var DefaultPrinter = Printer{
 	},
 }
 
-func (p *Printer) SetSchema(schema llx.Schema) {
+func (p *Printer) SetSchema(schema resources.ResourcesSchema) {
 	p.schema = schema
 }
 

diff --git a/cli/shell/completer.go b/cli/shell/completer.go
@@ -6,21 +6,21 @@ package shell
 import (
 	"github.com/c-bata/go-prompt"
 	"go.mondoo.com/cnquery/v10"
-	"go.mondoo.com/cnquery/v10/llx"
 	"go.mondoo.com/cnquery/v10/mqlc"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 )
 
 var completerSeparator = string([]byte{'.', ' '})
 
 // Completer is an auto-complete helper for the shell
 type Completer struct {
-	schema      llx.Schema
+	schema      resources.ResourcesSchema
 	features    cnquery.Features
 	queryPrefix func() string
 }
 
 // NewCompleter creates a new Mondoo completer object
-func NewCompleter(schema llx.Schema, features cnquery.Features, queryPrefix func() string) *Completer {
+func NewCompleter(schema resources.ResourcesSchema, features cnquery.Features, queryPrefix func() string) *Completer {
 	return &Completer{
 		schema:      schema,
 		features:    features,

diff --git a/explorer/bundle.go b/explorer/bundle.go
@@ -15,9 +15,9 @@ import (
 	"github.com/segmentio/ksuid"
 	"go.mondoo.com/cnquery/v10"
 	"go.mondoo.com/cnquery/v10/checksums"
-	llx "go.mondoo.com/cnquery/v10/llx"
 	"go.mondoo.com/cnquery/v10/mqlc"
 	"go.mondoo.com/cnquery/v10/mrn"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 	"go.mondoo.com/cnquery/v10/utils/multierr"
 	"sigs.k8s.io/yaml"
 )
@@ -212,7 +212,7 @@ func (p *Bundle) AddBundle(other *Bundle) error {
 }
 
 // Compile a bundle. See CompileExt for a full description.
-func (p *Bundle) Compile(ctx context.Context, schema llx.Schema) (*BundleMap, error) {
+func (p *Bundle) Compile(ctx context.Context, schema resources.ResourcesSchema) (*BundleMap, error) {
 	return p.CompileExt(ctx, BundleCompileConf{
 		CompilerConfig: mqlc.NewConfig(schema, cnquery.DefaultFeatures),
 	})

diff --git a/explorer/executor/mustcompile.go b/explorer/executor/mustcompile.go
@@ -12,7 +12,7 @@ import (
 
 func MustCompile(code string) *llx.CodeBundle {
 	codeBundle, err := mqlc.Compile(code, nil,
-		mqlc.NewConfig(providers.DefaultRuntime().Schema(), cnquery.DefaultFeatures))
+		mqlc.NewConfig(providers.Coordinator.Schema(), cnquery.DefaultFeatures))
 	if err != nil {
 		panic(err)
 	}

diff --git a/explorer/query_conductor.go b/explorer/query_conductor.go
@@ -14,6 +14,7 @@ import (
 	llx "go.mondoo.com/cnquery/v10/llx"
 	"go.mondoo.com/cnquery/v10/mqlc"
 	"go.mondoo.com/cnquery/v10/mrn"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 	"go.mondoo.com/cnquery/v10/utils/multierr"
 	"go.mondoo.com/ranger-rpc/codes"
 	"go.mondoo.com/ranger-rpc/status"
@@ -279,7 +280,7 @@ func (s *LocalServices) addQueryToJob(ctx context.Context, query *Mquery, job *E
 
 // MatchFilters will take the list of filters and only return the ones
 // that are supported by the given querypacks.
-func MatchFilters(entityMrn string, filters []*Mquery, packs []*QueryPack, schema llx.Schema) (string, error) {
+func MatchFilters(entityMrn string, filters []*Mquery, packs []*QueryPack, schema resources.ResourcesSchema) (string, error) {
 	supported := map[string]*Mquery{}
 	for i := range packs {
 		pack := packs[i]

diff --git a/explorer/scan/fetcher.go b/explorer/scan/fetcher.go
@@ -11,7 +11,7 @@ import (
 
 	"go.mondoo.com/cnquery/v10"
 	"go.mondoo.com/cnquery/v10/explorer"
-	"go.mondoo.com/cnquery/v10/llx"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 	"go.mondoo.com/cnquery/v10/utils/multierr"
 )
 
@@ -25,7 +25,7 @@ func newFetcher() *fetcher {
 	}
 }
 
-func (f *fetcher) fetchBundles(ctx context.Context, schema llx.Schema, urls ...string) (*explorer.Bundle, error) {
+func (f *fetcher) fetchBundles(ctx context.Context, schema resources.ResourcesSchema, urls ...string) (*explorer.Bundle, error) {
 	var res *explorer.Bundle = &explorer.Bundle{}
 
 	for i := range urls {

diff --git a/llx/runtime.go b/llx/runtime.go
@@ -14,18 +14,12 @@ type Runtime interface {
 	CreateResource(name string, args map[string]*Primitive) (Resource, error)
 	CloneResource(src Resource, id string, fields []string, args map[string]*Primitive) (Resource, error)
 	WatchAndUpdate(resource Resource, field string, watcherUID string, callback func(res interface{}, err error)) error
-	Schema() Schema
+	Schema() resources.ResourcesSchema
 	Close()
 	Recording() Recording
 	SetRecording(recording Recording) error
 }
 
-type Schema interface {
-	Lookup(resource string) *resources.ResourceInfo
-	LookupField(resource string, field string) (*resources.ResourceInfo, *resources.Field)
-	AllResources() map[string]*resources.ResourceInfo
-}
-
 type Recording interface {
 	Save() error
 	EnsureAsset(asset *inventory.Asset, provider string, connectionID uint32, conf *inventory.Config)

diff --git a/mql/internal/builder.go b/mql/internal/builder.go
@@ -13,6 +13,7 @@ import (
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/v10"
 	"go.mondoo.com/cnquery/v10/llx"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 )
 
 type query struct {
@@ -102,7 +103,7 @@ func (b *GraphBuilder) WithFeatureBoolAssertions(featureBoolAssertions bool) {
 	b.featureBoolAssertions = featureBoolAssertions
 }
 
-func (b *GraphBuilder) Build(schema llx.Schema, runtime llx.Runtime, assetMrn string) (*GraphExecutor, error) {
+func (b *GraphBuilder) Build(schema resources.ResourcesSchema, runtime llx.Runtime, assetMrn string) (*GraphExecutor, error) {
 	resultChan := make(chan *llx.RawResult, 128)
 
 	queries := make(map[string]query, len(b.queries))

diff --git a/mql/internal/execution_manager.go b/mql/internal/execution_manager.go
@@ -10,10 +10,11 @@ import (
 
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/v10/llx"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 )
 
 type executionManager struct {
-	schema  llx.Schema
+	schema  resources.ResourcesSchema
 	runtime llx.Runtime
 	// runQueue is the channel the execution manager will read
 	// items that need to be run from
@@ -37,7 +38,7 @@ type runQueueItem struct {
 	props      map[string]*llx.Result
 }
 
-func newExecutionManager(schema llx.Schema, runtime llx.Runtime, runQueue chan runQueueItem,
+func newExecutionManager(schema resources.ResourcesSchema, runtime llx.Runtime, runQueue chan runQueueItem,
 	resultChan chan *llx.RawResult, timeout time.Duration,
 ) *executionManager {
 	return &executionManager{

diff --git a/mqlc/labels.go b/mqlc/labels.go
@@ -9,12 +9,13 @@ import (
 	"strconv"
 
 	"go.mondoo.com/cnquery/v10/llx"
+	"go.mondoo.com/cnquery/v10/providers-sdk/v1/resources"
 	"go.mondoo.com/cnquery/v10/types"
 	"golang.org/x/text/transform"
 	"golang.org/x/text/unicode/norm"
 )
 
-func createLabel(res *llx.CodeBundle, ref uint64, schema llx.Schema) (string, error) {
+func createLabel(res *llx.CodeBundle, ref uint64, schema resources.ResourcesSchema) (string, error) {
 	code := res.CodeV2
 	chunk := code.Chunk(ref)
 
@@ -129,7 +130,7 @@ func stripCtlAndExtFromUnicode(str string) string {
 }
 
 // UpdateLabels for the given code under the schema
-func UpdateLabels(res *llx.CodeBundle, schema llx.Schema) error {
+func UpdateLabels(res *llx.CodeBundle, schema resources.ResourcesSchema) error {
 	if res == nil || res.CodeV2 == nil {
 		return errors.New("cannot create labels without code")
 	}
@@ -147,7 +148,7 @@ func UpdateLabels(res *llx.CodeBundle, schema llx.Schema) error {
 	return nil
 }
 
-func updateLabels(res *llx.CodeBundle, block *llx.Block, schema llx.Schema) error {
+func updateLabels(res *llx.CodeBundle, block *llx.Block, schema resources.ResourcesSchema) error {
 	datapoints := block.Datapoints
 	code := res.CodeV2
 	labels := res.Labels.Labels

diff --git a/mqlc/mqlc.go b/mqlc/mqlc.go
@@ -87,7 +87,7 @@ func (vm *varmap) len() int {
 }
 
 type CompilerConfig struct {
-	Schema          llx.Schema
+	Schema          resources.ResourcesSchema
 	UseAssetContext bool
 	Stats           CompilerStats
 }
@@ -100,7 +100,7 @@ func (c *CompilerConfig) EnableMultiStats() {
 	c.Stats = newCompilerMultiStats()
 }
 
-func NewConfig(schema llx.Schema, features cnquery.Features) CompilerConfig {
+func NewConfig(schema resources.ResourcesSchema, features cnquery.Features) CompilerConfig {
 	return CompilerConfig{
 		Schema:          schema,
 		UseAssetContext: features.IsActive(cnquery.MQLAssetContext),
@@ -203,7 +203,7 @@ func findFuzzy(name string, names []string) fuzzy.Ranks {
 	return suggested
 }
 
-func addResourceSuggestions(schema llx.Schema, name string, res *llx.CodeBundle) {
+func addResourceSuggestions(schema resources.ResourcesSchema, name string, res *llx.CodeBundle) {
 	resourceInfos := schema.AllResources()
 	names := make([]string, len(resourceInfos))
 	i := 0
@@ -290,7 +290,7 @@ func addFieldSuggestions(fields map[string]llx.Documentation, fieldName string,
 // 	return typ
 // }
 
-func blockCallType(typ types.Type, schema llx.Schema) types.Type {
+func blockCallType(typ types.Type, schema resources.ResourcesSchema) types.Type {
 	if typ.IsArray() {
 		return types.Array(types.Block)
 	}
@@ -2171,7 +2171,7 @@ func (c *compiler) CompileParsed(ast *parser.AST) error {
 	return nil
 }
 
-func getMinMondooVersion(schema llx.Schema, current string, resource string, field string) string {
+func getMinMondooVersion(schema resources.ResourcesSchema, current string, resource string, field string) string {
 	info := schema.Lookup(resource)
 	if info == nil {
 		return current
@@ -2242,7 +2242,6 @@ func CompileAST(ast *parser.AST, props map[string]*llx.Primitive, conf CompilerC
 
 // Compile a code piece against a schema into chunky code
 func compile(input string, props map[string]*llx.Primitive, compilerConf CompilerConfig) (*llx.CodeBundle, error) {
-
 	// remove leading whitespace; we are re-using this later on
 	input = Dedent(input)