From 5602c920be0a40ea38f1087ba6fe6cec6169e2ac Mon Sep 17 00:00:00 2001 From: Dominik Richter Date: Thu, 14 Sep 2023 16:15:29 -0700 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20add=20configuration=20for=20prov?= =?UTF-8?q?iders=20in=20dev=20env?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This has now happened a few too many times: You get a PR with a change to a provider, but forgot to update your builtin.go file. After 30min you realize you have been testing the providers on disk, instead of the ones in the repo. Also configuring providers is a bit annoying (with all the things you need to change in `builtin.go`). This is now streamlined in `make providers/config`. You can configure it in `providers.yaml` in the root folder. Signed-off-by: Dominik Richter --- Makefile | 7 +- providers-sdk/v1/util/configure/configure.go | 134 +++++++++++++++++++ providers.yaml | 15 +++ providers/builtin.go | 82 +----------- providers/coordinator.go | 8 ++ providers/providers.go | 17 +++ 6 files changed, 182 insertions(+), 81 deletions(-) create mode 100644 providers-sdk/v1/util/configure/configure.go create mode 100644 providers.yaml diff --git a/Makefile b/Makefile index 9845e9e2db..950781139e 100644 --- a/Makefile +++ b/Makefile @@ -135,7 +135,7 @@ define gomodtidyProvider endef .PHONY: providers -providers: providers/proto providers/build +providers: providers/proto providers/config providers/build .PHONY: providers/proto providers/proto: @@ -145,6 +145,11 @@ providers/proto: go generate ./providers-sdk/v1/inventory go generate ./providers-sdk/v1/plugin +.PHONY: providers/config +providers/config: + go run ./providers-sdk/v1/util/configure/configure.go -f providers.yaml -o providers/builtin.go + gofmt -w providers/builtin.go + .PHONY: providers/lr providers/lr: go build -o lr ./providers-sdk/v1/lr/cli/main.go diff --git a/providers-sdk/v1/util/configure/configure.go b/providers-sdk/v1/util/configure/configure.go new file mode 100644 index 0000000000..7d765a6834 --- /dev/null +++ b/providers-sdk/v1/util/configure/configure.go @@ -0,0 +1,134 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package main + +import ( + "fmt" + "os" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + "go.mondoo.com/cnquery/logger" + "sigs.k8s.io/yaml" +) + +type ProvidersConf struct { + Builtin []string `json:"builtin"` +} + +var rootCmd = &cobra.Command{ + Use: "configure [-f config] [-o file]", + Short: "configure providers for cnquery", + Run: func(cmd *cobra.Command, args []string) { + confPath, err := cmd.Flags().GetString("file") + if err != nil { + log.Fatal().Err(err).Msg("Can't get --file") + } + outPath, err := cmd.Flags().GetString("output") + if err != nil { + log.Fatal().Err(err).Msg("Can't get --output") + } + + raw, err := os.ReadFile(confPath) + if err != nil { + log.Fatal().Err(err).Str("path", confPath).Msg("failed to read config file") + } + + var conf ProvidersConf + err = yaml.Unmarshal(raw, &conf) + if err != nil { + log.Fatal().Err(err).Str("path", confPath).Msg("failed to parse config file") + } + + builtinGo, err := genBuiltinGo(conf) + + if err = os.WriteFile(outPath, []byte(builtinGo), 0o644); err != nil { + log.Fatal().Err(err).Str("path", outPath).Msg("failed to write output") + } + log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("configured builtin providers") + }, +} + +func genBuiltinGo(conf ProvidersConf) (string, error) { + var imports string + var infos string + var configs string + + for _, provider := range conf.Builtin { + imports += fmt.Sprintf("\t%sconf \"go.mondoo.com/cnquery/providers/%s/config\"\n", provider, provider) + imports += fmt.Sprintf("\t%s \"go.mondoo.com/cnquery/providers/%s/provider\"\n", provider, provider) + infos += fmt.Sprintf( + "//go:embed %s/resources/%s.resources.json\n"+ + "var %sInfo []byte\n", + provider, provider, provider) + configs += fmt.Sprintf(` + %sconf.Config.ID: { + Runtime: &RunningProvider{ + Name: %sconf.Config.Name, + ID: %sconf.Config.ID, + Plugin: %s.Init(), + Schema: MustLoadSchema("%s", %sInfo), + isClosed: false, + }, + Config: &%sconf.Config, + }, +`, provider, provider, provider, provider, provider, provider, provider) + } + + return fmt.Sprintf(template, imports, infos, configs), nil +} + +const template = `// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 +// +// This file is auto-generated by 'make providers/config' + +package providers + +// Uncomment any provider you want to load directly into the binary. +// This is primarily useful for debugging purposes, if you want to +// trace into any provider without having to debug the plugin +// connection separately. + +import ( + _ "embed" + + coreconf "go.mondoo.com/cnquery/providers/core/config" + core "go.mondoo.com/cnquery/providers/core/provider" +%s) + +//go:embed core/resources/core.resources.json +var coreInfo []byte + +%s +var builtinProviders = map[string]*builtinProvider{ + coreconf.Config.ID: { + Runtime: &RunningProvider{ + Name: coreconf.Config.Name, + ID: coreconf.Config.ID, + Plugin: core.Init(), + Schema: MustLoadSchema("core", coreInfo), + isClosed: false, + }, + Config: &coreconf.Config, + }, +%s +} +` + +func init() { + rootCmd.Flags().StringP("file", "f", "providers.yaml", "config file for providers") + rootCmd.Flags().StringP("output", "o", "providers/builtin.go", "output builtin.go file") +} + +func main() { + logger.CliCompactLogger(logger.LogOutputWriter) + zerolog.SetGlobalLevel(zerolog.InfoLevel) + + if err := rootCmd.Execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/providers.yaml b/providers.yaml new file mode 100644 index 0000000000..3379838386 --- /dev/null +++ b/providers.yaml @@ -0,0 +1,15 @@ +# Copyright (c) Mondoo, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# Configure builtin providers for the dev environment. +# This only covers additional builtin providers, ie those +# that are normally not builtin. Things like `core` are +# always builtin. +# +# Builtin providers are great testing. Example: if you +# review a PR where someone changed the `os` provider, +# you'll want to set it as builtin for fast testing. +# In this case it will be pulled from your local repo +# instead of using the pre-installed provider in +# your OS (like ~/.mondoo/providers/os). +builtin: [] diff --git a/providers/builtin.go b/providers/builtin.go index b05114897a..0ea4152d8a 100644 --- a/providers/builtin.go +++ b/providers/builtin.go @@ -1,5 +1,7 @@ // Copyright (c) Mondoo, Inc. // SPDX-License-Identifier: BUSL-1.1 +// +// This file is auto-generated by 'make providers/config' package providers @@ -10,34 +12,14 @@ package providers import ( _ "embed" - "encoding/json" - osfs "os" - "go.mondoo.com/cnquery/providers-sdk/v1/plugin" - "go.mondoo.com/cnquery/providers-sdk/v1/resources" coreconf "go.mondoo.com/cnquery/providers/core/config" core "go.mondoo.com/cnquery/providers/core/provider" - // osconf "go.mondoo.com/cnquery/providers/os/config" - // os "go.mondoo.com/cnquery/providers/os/provider" ) -var BuiltinCoreID = coreconf.Config.ID - //go:embed core/resources/core.resources.json var coreInfo []byte -// //go:embed os/resources/os.resources.json -// var osInfo []byte - -// //go:embed network/resources/network.resources.json -// var networkInfo []byte - -// //go:embed k8s/resources/k8s.resources.json -// var k8sInfo []byte - -// //go:embed azure.resources.json -// var azureInfo []byte - var builtinProviders = map[string]*builtinProvider{ coreconf.Config.ID: { Runtime: &RunningProvider{ @@ -49,64 +31,4 @@ var builtinProviders = map[string]*builtinProvider{ }, Config: &coreconf.Config, }, - // osconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: osconf.Config.Name, - // ID: osconf.Config.ID, - // Plugin: os.Init(), - // Schema: MustLoadSchema("os", osInfo), - // isClosed: false, - // }, - // Config: &osconf.Config, - // }, - // networkconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: networkconf.Config.Name, - // ID: networkconf.Config.ID, - // Plugin: network.Init(), - // Schema: MustLoadSchema("network", networkInfo), - // isClosed: false, - // }, - // Config: &networkconf.Config, - // }, - // k8sconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: k8sconf.Config.Name, - // ID: k8sconf.Config.ID, - // Plugin: k8s.Init(), - // Schema: MustLoadSchema("k8s", k8sInfo), - // isClosed: false, - // }, - // Config: &k8sconf.Config, - // }, - // azureconf.Config.ID: { - // Runtime: &RunningProvider{ - // Name: azureconf.Config.Name, - // ID: azureconf.Config.ID, - // Plugin: azure.Init(), - // Schema: MustLoadSchema("azure", azureInfo), - // isClosed: false, - // }, - // Config: &azureconf.Config, -} - -type builtinProvider struct { - Runtime *RunningProvider - Config *plugin.Provider -} - -func MustLoadSchema(name string, data []byte) *resources.Schema { - var res resources.Schema - if err := json.Unmarshal(data, &res); err != nil { - panic("failed to embed schema for " + name) - } - return &res -} - -func MustLoadSchemaFromFile(name string, path string) *resources.Schema { - raw, err := osfs.ReadFile(path) - if err != nil { - panic("cannot read schema file: " + path) - } - return MustLoadSchema(name, raw) } diff --git a/providers/coordinator.go b/providers/coordinator.go index 0bd952eb06..f28976c6fb 100644 --- a/providers/coordinator.go +++ b/providers/coordinator.go @@ -15,9 +15,12 @@ import ( "github.com/rs/zerolog/log" pp "go.mondoo.com/cnquery/providers-sdk/v1/plugin" "go.mondoo.com/cnquery/providers-sdk/v1/resources" + coreconf "go.mondoo.com/cnquery/providers/core/config" "go.mondoo.com/cnquery/providers/core/resources/versions/semver" ) +var BuiltinCoreID = coreconf.Config.ID + var Coordinator = coordinator{ Running: []*RunningProvider{}, } @@ -28,6 +31,11 @@ type coordinator struct { mutex sync.Mutex } +type builtinProvider struct { + Runtime *RunningProvider + Config *pp.Provider +} + type RunningProvider struct { Name string ID string diff --git a/providers/providers.go b/providers/providers.go index cf8ee88913..79f6ccb64c 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "os" + osfs "os" "path/filepath" "runtime" "strings" @@ -567,3 +568,19 @@ func (p Providers) Add(nu *Provider) { p[nu.ID] = nu } } + +func MustLoadSchema(name string, data []byte) *resources.Schema { + var res resources.Schema + if err := json.Unmarshal(data, &res); err != nil { + panic("failed to embed schema for " + name) + } + return &res +} + +func MustLoadSchemaFromFile(name string, path string) *resources.Schema { + raw, err := osfs.ReadFile(path) + if err != nil { + panic("cannot read schema file: " + path) + } + return MustLoadSchema(name, raw) +}