From 2d9440292bf66f10a9e11662889032f62983cb24 Mon Sep 17 00:00:00 2001 From: Mikita Iwanowski Date: Fri, 4 Oct 2024 15:30:49 +0200 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=EF=B8=8F=20remote=20builtin=20provide?= =?UTF-8?q?rs=20(#4710)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: built-in remote providers * feat: add DO NOT EDIT * improve logging * feat: handle tilda shortcut * LICENSE * edit-provider: handle remove providers & marshal builtin * update help msg * resolve nit --- .vscode/launch.json | 11 ++ Makefile | 2 +- providers-sdk/v1/util/configure/configure.go | 90 ++++++++++------ .../v1/util/configure/provider_conf.go | 102 ++++++++++++++++++ providers.yaml | 7 ++ providers/builtin_dev.go | 2 +- 6 files changed, 178 insertions(+), 36 deletions(-) create mode 100644 providers-sdk/v1/util/configure/provider_conf.go diff --git a/.vscode/launch.json b/.vscode/launch.json index d39c877deb..85931b3a76 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -197,6 +197,17 @@ "args": [ "shell", "ssh", "user@18.215.249.49", ], + }, + { + "name": "Configure Built-in Providers", + "type": "go", + "request": "launch", + "program": "${workspaceRoot}/providers-sdk/v1/util/configure", + "cwd": "${workspaceRoot}", + "args": [ + "-f", "${workspaceRoot}/providers.yaml", + "-o", "${workspaceRoot}/providers/builtin_dev.go" + ] } ] } diff --git a/Makefile b/Makefile index 28cad96e22..2266fe9e0f 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,7 @@ providers/proto: .PHONY: providers/config providers/config: - go run ./providers-sdk/v1/util/configure/configure.go -f providers.yaml -o providers/builtin_dev.go + go run ./providers-sdk/v1/util/configure -f providers.yaml -o providers/builtin_dev.go .PHONY: providers/defaults providers/defaults: diff --git a/providers-sdk/v1/util/configure/configure.go b/providers-sdk/v1/util/configure/configure.go index bbabc341e6..75218357e2 100644 --- a/providers-sdk/v1/util/configure/configure.go +++ b/providers-sdk/v1/util/configure/configure.go @@ -23,10 +23,6 @@ import ( "sigs.k8s.io/yaml" ) -type ProvidersConf struct { - Builtin []string `json:"builtin"` -} - func init() { rootCmd.Flags().StringP("file", "f", "providers.yaml", "config file for providers") rootCmd.Flags().StringP("output", "o", "providers/builtin_dev.go", "output go-file for builtin dev providers") @@ -77,18 +73,18 @@ var rootCmd = &cobra.Command{ if err = os.WriteFile(outPath, builtinGo, 0o644); err != nil { log.Fatal().Err(err).Str("path", outPath).Msg("failed to write output") } - log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(1/3) configured builtin providers") + log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(1/3) configured builtin providers") buildProviders(conf.Builtin) - log.Info().Strs("providers", conf.Builtin).Msg("(2/3) built providers") + log.Info().Strs("providers", conf.Providers()).Msg("(2/3) built providers") rewireDependencies(conf.Builtin) - log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(3/3) rewired dependencies/files") + log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(3/3) rewired dependencies/files") }, } var editProvidersCmd = &cobra.Command{ - Use: "edit-providers NAME1 NAME2 [-f config]", + Use: "edit-providers NAME1 NAME2:REMOTE:GOPACKAGE [-f config]", Short: "adds a provider to the config", Run: func(cmd *cobra.Command, args []string) { confPath, err := cmd.Flags().GetString("file") @@ -108,15 +104,29 @@ var editProvidersCmd = &cobra.Command{ } if len(conf.Builtin) > 0 { - log.Warn().Strs("providers", conf.Builtin).Msg("overwrite existing providers in config") + log.Warn().Strs("providers", conf.Providers()).Msg("overwrite existing providers in config") } // set new providers - conf.Builtin = args - slices.Sort(conf.Builtin) + for _, arg := range args { + if !strings.Contains(arg, ":") { + conf.Builtin = append(conf.Builtin, Builtin{Name: arg}) + continue + } + + parts := strings.Split(arg, ":") + if len(parts) != 3 { + log.Fatal().Str("provider", arg).Msg("invalid provider format, must be NAME:REMOTE:GOPACKAGE") + } + + conf.Builtin = append(conf.Builtin, Builtin{Name: parts[0], Remote: parts[1], GoPackage: parts[2]}) + } + slices.SortFunc(conf.Builtin, func(a, b Builtin) int { + return strings.Compare(a.Name, b.Name) + }) conf.Builtin = slices.Compact(conf.Builtin) - log.Info().Strs("providers", conf.Builtin).Msg("configured providers") + log.Info().Strs("providers", conf.Providers()).Msg("configured providers") raw, err = yaml.Marshal(conf) if err != nil { @@ -137,13 +147,15 @@ func genBuiltinGo(conf ProvidersConf) ([]byte, error) { for _, provider := range conf.Builtin { // imports cannot contain dashes - trimProvider := strings.Replace(provider, "-", "", -1) - imports += fmt.Sprintf("\t%sconf \"go.mondoo.com/cnquery/v11/providers/%s/config\"\n", trimProvider, provider) - imports += fmt.Sprintf("\t%s \"go.mondoo.com/cnquery/v11/providers/%s/provider\"\n", trimProvider, provider) + trimProvider := strings.Replace(provider.Name, "-", "", -1) + imports += fmt.Sprintf("\t%sconf \"%s/config\"\n", trimProvider, provider.GoPackage) + imports += fmt.Sprintf("\t%s \"%s/provider\"\n", trimProvider, provider.GoPackage) + infos += fmt.Sprintf( "//go:embed %s.resources.json\n"+ "var %sInfo []byte\n", - provider, trimProvider) + provider.Name, trimProvider) + configs += fmt.Sprintf(` builtinProviders[%sconf.Config.ID] = &builtinProvider{ Runtime: &RunningProvider{ @@ -155,7 +167,7 @@ func genBuiltinGo(conf ProvidersConf) ([]byte, error) { }, Config: &%sconf.Config, } -`, trimProvider, trimProvider, trimProvider, trimProvider, provider, trimProvider, trimProvider) +`, trimProvider, trimProvider, trimProvider, trimProvider, provider.Name, trimProvider, trimProvider) } res := fmt.Sprintf(template, imports, infos, configs) @@ -166,7 +178,7 @@ const template = `// Copyright (c) Mondoo, Inc. // SPDX-License-Identifier: BUSL-1.1 // // This file is auto-generated by 'make providers/config' -// and configured via 'providers.yaml' +// and configured via 'providers.yaml'; DO NOT EDIT. package providers @@ -196,26 +208,31 @@ func init() { } ` -func buildProviders(providers []string) { +func buildProviders(providers []Builtin) { for i, provider := range providers { - cmd := exec.Command("make", "providers/build/"+provider) + cmd := exec.Command("make", "providers/build/"+provider.Name) + if provider.Remote != "" { + cmd = exec.Command("make", "provider/generate") + cmd.Dir = provider.Remote + } + var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out - log.Debug().Str("provider", provider).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers))) + log.Debug().Str("provider", provider.Name).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers))) if err := cmd.Run(); err != nil { fmt.Println(out.String()) - log.Error().Err(err).Str("provider", provider).Msg("failed to build provider") + log.Error().Err(err).Str("provider", provider.Name).Msg("failed to build provider") } // inefficient copy... - src := "providers/" + provider + "/resources/" + provider + ".resources.json" + src := provider.Resource() raw, err := os.ReadFile(src) if err != nil { log.Fatal().Err(err).Str("src", src).Msg("failed to read resources json") } - dst := "providers/" + provider + ".resources.json" + dst := provider.Dist() err = os.WriteFile(dst, raw, 0o644) if err != nil { log.Fatal().Err(err).Str("dst", dst).Msg("failed to write resources json") @@ -228,7 +245,7 @@ var ( reBuiltinDep = regexp.MustCompile(`go.mondoo.com/cnquery/v11/providers/.*`) ) -func rewireDependencies(providers []string) { +func rewireDependencies(providers []Builtin) { raw, err := os.ReadFile("go.mod") if err != nil { log.Fatal().Err(err).Msg("failed to read go.mod") @@ -241,28 +258,33 @@ func rewireDependencies(providers []string) { deps := "" replace := "" for _, provider := range providers { - _, err := os.Stat("providers/" + provider + "/go.mod") + goModPath := provider.GoMod() + _, err := os.Stat(goModPath) if err != nil { if errors.Is(err, os.ErrNotExist) { - log.Info().Str("provider", provider).Msg("skipping provider without go.mod") + log.Info().Str("provider", provider.Name).Msg("skipping provider without go.mod") continue } else { - log.Fatal().Err(err).Str("provider", provider).Msg("failed to stat provider go.mod") + log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to stat provider go.mod") } } - goModContent, err := os.ReadFile("providers/" + provider + "/go.mod") + goModContent, err := os.ReadFile(goModPath) if err != nil { - log.Fatal().Err(err).Str("provider", provider).Msg("failed to read provider go.mod") + log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to read provider go.mod") } - goMod, err := modfile.Parse(fmt.Sprintf("%s/go.mod", provider), goModContent, nil) + goMod, err := modfile.Parse(goModPath, goModContent, nil) if err != nil { - log.Fatal().Err(err).Str("provider", provider).Msg("failed to parse provider go.mod") + log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to parse provider go.mod") } // we don't care about the specific version for dev - deps += "\n\tgo.mondoo.com/cnquery/v11/providers/" + provider + " v0.0.0" - replace += "\nreplace go.mondoo.com/cnquery/v11/providers/" + provider + " => ./providers/" + provider + deps += "\n\t" + provider.GoPackage + " v0.0.0" + if provider.Remote != "" { + replace += "\nreplace " + provider.GoPackage + " => " + provider.Remote + } else { + replace += "\nreplace " + provider.GoPackage + " => ./providers/" + provider.Name + } // if the provider has any specific pinned replacements, we also add those to allow compiling for _, r := range goMod.Replace { // special case: we don't want to pull in provider's 'replace go.mondoo.com/cnquery/v11 => ../..' in. diff --git a/providers-sdk/v1/util/configure/provider_conf.go b/providers-sdk/v1/util/configure/provider_conf.go new file mode 100644 index 0000000000..762b8959b0 --- /dev/null +++ b/providers-sdk/v1/util/configure/provider_conf.go @@ -0,0 +1,102 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 +package main + +import ( + "encoding/json" + "fmt" + "os" + "strings" +) + +type ProvidersConf struct { + Builtin []Builtin `json:"builtin"` + + providers []string // Providers names +} + +func (c ProvidersConf) Providers() []string { + if len(c.providers) == 0 { + for _, b := range c.Builtin { + c.providers = append(c.providers, b.Name) + } + } + return c.providers +} + +type Builtin struct { + Name string + Remote string + GoPackage string +} + +func (b Builtin) Cwd() string { + if b.Remote != "" { + return b.Remote + } + + return "" +} + +func (b Builtin) Resource() string { + if b.Remote != "" { + return fmt.Sprintf("%s/resources/%s.resources.json", b.Remote, b.Name) + } + return "providers/" + b.Name + "/resources/" + b.Name + ".resources.json" +} + +func (b Builtin) Dist() string { + return "providers/" + b.Name + ".resources.json" +} + +func (b Builtin) GoMod() string { + if b.Remote != "" { + return b.Remote + "/go.mod" + } + return "providers/" + b.Name + "/go.mod" +} + +func (b *Builtin) UnmarshalJSON(data []byte) error { + var name string + if err := json.Unmarshal(data, &name); err == nil { + b.Name = name + b.GoPackage = "go.mondoo.com/cnquery/v11/providers/" + name + + return nil + } + + var raw struct { + Name string `json:"name"` + Remote string `json:"remote"` + GoPackage string `json:"goPackage"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + b.Name = raw.Name + b.Remote = raw.Remote + if strings.HasPrefix(b.Remote, "~/") { + b.Remote = os.ExpandEnv(strings.Replace(b.Remote, "~/", "$HOME/", 1)) + } + + b.GoPackage = raw.GoPackage + + return nil +} + +func (b *Builtin) MarshalJSON() ([]byte, error) { + if b.Remote == "" && b.GoPackage == "" { + return json.Marshal(b.Name) + } + + return json.Marshal(struct { + Name string `json:"name"` + Remote string `json:"remote"` + GoPackage string `json:"goPackage"` + }{ + Name: b.Name, + Remote: b.Remote, + GoPackage: b.GoPackage, + }) +} diff --git a/providers.yaml b/providers.yaml index 3379838386..bb0e34dc1b 100644 --- a/providers.yaml +++ b/providers.yaml @@ -12,4 +12,11 @@ # In this case it will be pulled from your local repo # instead of using the pre-installed provider in # your OS (like ~/.mondoo/providers/os). +# +# If you want to test a provider that is not part of +# the cnquery repo, you can add it here as well. +# Example: +# - name: my-awesome-provider +# remote: ~/path/to/my-awesome-provider +# goPackage: github.com/me/my-awesome-provider builtin: [] diff --git a/providers/builtin_dev.go b/providers/builtin_dev.go index 1aa179caad..110fa9e26b 100644 --- a/providers/builtin_dev.go +++ b/providers/builtin_dev.go @@ -2,7 +2,7 @@ // SPDX-License-Identifier: BUSL-1.1 // // This file is auto-generated by 'make providers/config' -// and configured via 'providers.yaml' +// and configured via 'providers.yaml'; DO NOT EDIT. package providers