Skip to content

Commit

Permalink
⭐️ remote builtin providers (#4710)
Browse files Browse the repository at this point in the history
* feat: built-in remote providers

* feat: add DO NOT EDIT

* improve logging

* feat: handle tilda shortcut

* LICENSE

* edit-provider: handle remove providers & marshal builtin

* update help msg

* resolve nit
  • Loading branch information
slntopp authored Oct 4, 2024
1 parent 5788fb4 commit 2d94402
Show file tree
Hide file tree
Showing 6 changed files with 178 additions and 36 deletions.
11 changes: 11 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,17 @@
"args": [
"shell", "ssh", "[email protected]",
],
},
{
"name": "Configure Built-in Providers",
"type": "go",
"request": "launch",
"program": "${workspaceRoot}/providers-sdk/v1/util/configure",
"cwd": "${workspaceRoot}",
"args": [
"-f", "${workspaceRoot}/providers.yaml",
"-o", "${workspaceRoot}/providers/builtin_dev.go"
]
}
]
}
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ providers/proto:

.PHONY: providers/config
providers/config:
go run ./providers-sdk/v1/util/configure/configure.go -f providers.yaml -o providers/builtin_dev.go
go run ./providers-sdk/v1/util/configure -f providers.yaml -o providers/builtin_dev.go

.PHONY: providers/defaults
providers/defaults:
Expand Down
90 changes: 56 additions & 34 deletions providers-sdk/v1/util/configure/configure.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,6 @@ import (
"sigs.k8s.io/yaml"
)

type ProvidersConf struct {
Builtin []string `json:"builtin"`
}

func init() {
rootCmd.Flags().StringP("file", "f", "providers.yaml", "config file for providers")
rootCmd.Flags().StringP("output", "o", "providers/builtin_dev.go", "output go-file for builtin dev providers")
Expand Down Expand Up @@ -77,18 +73,18 @@ var rootCmd = &cobra.Command{
if err = os.WriteFile(outPath, builtinGo, 0o644); err != nil {
log.Fatal().Err(err).Str("path", outPath).Msg("failed to write output")
}
log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(1/3) configured builtin providers")
log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(1/3) configured builtin providers")

buildProviders(conf.Builtin)
log.Info().Strs("providers", conf.Builtin).Msg("(2/3) built providers")
log.Info().Strs("providers", conf.Providers()).Msg("(2/3) built providers")

rewireDependencies(conf.Builtin)
log.Info().Str("path", outPath).Strs("providers", conf.Builtin).Msg("(3/3) rewired dependencies/files")
log.Info().Str("path", outPath).Strs("providers", conf.Providers()).Msg("(3/3) rewired dependencies/files")
},
}

var editProvidersCmd = &cobra.Command{
Use: "edit-providers NAME1 NAME2 [-f config]",
Use: "edit-providers NAME1 NAME2:REMOTE:GOPACKAGE [-f config]",
Short: "adds a provider to the config",
Run: func(cmd *cobra.Command, args []string) {
confPath, err := cmd.Flags().GetString("file")
Expand All @@ -108,15 +104,29 @@ var editProvidersCmd = &cobra.Command{
}

if len(conf.Builtin) > 0 {
log.Warn().Strs("providers", conf.Builtin).Msg("overwrite existing providers in config")
log.Warn().Strs("providers", conf.Providers()).Msg("overwrite existing providers in config")
}

// set new providers
conf.Builtin = args
slices.Sort(conf.Builtin)
for _, arg := range args {
if !strings.Contains(arg, ":") {
conf.Builtin = append(conf.Builtin, Builtin{Name: arg})
continue
}

parts := strings.Split(arg, ":")
if len(parts) != 3 {
log.Fatal().Str("provider", arg).Msg("invalid provider format, must be NAME:REMOTE:GOPACKAGE")
}

conf.Builtin = append(conf.Builtin, Builtin{Name: parts[0], Remote: parts[1], GoPackage: parts[2]})
}
slices.SortFunc(conf.Builtin, func(a, b Builtin) int {
return strings.Compare(a.Name, b.Name)
})
conf.Builtin = slices.Compact(conf.Builtin)

log.Info().Strs("providers", conf.Builtin).Msg("configured providers")
log.Info().Strs("providers", conf.Providers()).Msg("configured providers")

raw, err = yaml.Marshal(conf)
if err != nil {
Expand All @@ -137,13 +147,15 @@ func genBuiltinGo(conf ProvidersConf) ([]byte, error) {

for _, provider := range conf.Builtin {
// imports cannot contain dashes
trimProvider := strings.Replace(provider, "-", "", -1)
imports += fmt.Sprintf("\t%sconf \"go.mondoo.com/cnquery/v11/providers/%s/config\"\n", trimProvider, provider)
imports += fmt.Sprintf("\t%s \"go.mondoo.com/cnquery/v11/providers/%s/provider\"\n", trimProvider, provider)
trimProvider := strings.Replace(provider.Name, "-", "", -1)
imports += fmt.Sprintf("\t%sconf \"%s/config\"\n", trimProvider, provider.GoPackage)
imports += fmt.Sprintf("\t%s \"%s/provider\"\n", trimProvider, provider.GoPackage)

infos += fmt.Sprintf(
"//go:embed %s.resources.json\n"+
"var %sInfo []byte\n",
provider, trimProvider)
provider.Name, trimProvider)

configs += fmt.Sprintf(`
builtinProviders[%sconf.Config.ID] = &builtinProvider{
Runtime: &RunningProvider{
Expand All @@ -155,7 +167,7 @@ func genBuiltinGo(conf ProvidersConf) ([]byte, error) {
},
Config: &%sconf.Config,
}
`, trimProvider, trimProvider, trimProvider, trimProvider, provider, trimProvider, trimProvider)
`, trimProvider, trimProvider, trimProvider, trimProvider, provider.Name, trimProvider, trimProvider)
}

res := fmt.Sprintf(template, imports, infos, configs)
Expand All @@ -166,7 +178,7 @@ const template = `// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1
//
// This file is auto-generated by 'make providers/config'
// and configured via 'providers.yaml'
// and configured via 'providers.yaml'; DO NOT EDIT.
package providers
Expand Down Expand Up @@ -196,26 +208,31 @@ func init() {
}
`

func buildProviders(providers []string) {
func buildProviders(providers []Builtin) {
for i, provider := range providers {
cmd := exec.Command("make", "providers/build/"+provider)
cmd := exec.Command("make", "providers/build/"+provider.Name)
if provider.Remote != "" {
cmd = exec.Command("make", "provider/generate")
cmd.Dir = provider.Remote
}

var out bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &out
log.Debug().Str("provider", provider).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers)))
log.Debug().Str("provider", provider.Name).Msg("build provider " + strconv.Itoa(i+1) + "/" + strconv.Itoa(len(providers)))
if err := cmd.Run(); err != nil {
fmt.Println(out.String())
log.Error().Err(err).Str("provider", provider).Msg("failed to build provider")
log.Error().Err(err).Str("provider", provider.Name).Msg("failed to build provider")
}

// inefficient copy...
src := "providers/" + provider + "/resources/" + provider + ".resources.json"
src := provider.Resource()
raw, err := os.ReadFile(src)
if err != nil {
log.Fatal().Err(err).Str("src", src).Msg("failed to read resources json")
}

dst := "providers/" + provider + ".resources.json"
dst := provider.Dist()
err = os.WriteFile(dst, raw, 0o644)
if err != nil {
log.Fatal().Err(err).Str("dst", dst).Msg("failed to write resources json")
Expand All @@ -228,7 +245,7 @@ var (
reBuiltinDep = regexp.MustCompile(`go.mondoo.com/cnquery/v11/providers/.*`)
)

func rewireDependencies(providers []string) {
func rewireDependencies(providers []Builtin) {
raw, err := os.ReadFile("go.mod")
if err != nil {
log.Fatal().Err(err).Msg("failed to read go.mod")
Expand All @@ -241,28 +258,33 @@ func rewireDependencies(providers []string) {
deps := ""
replace := ""
for _, provider := range providers {
_, err := os.Stat("providers/" + provider + "/go.mod")
goModPath := provider.GoMod()
_, err := os.Stat(goModPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
log.Info().Str("provider", provider).Msg("skipping provider without go.mod")
log.Info().Str("provider", provider.Name).Msg("skipping provider without go.mod")
continue
} else {
log.Fatal().Err(err).Str("provider", provider).Msg("failed to stat provider go.mod")
log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to stat provider go.mod")
}
}

goModContent, err := os.ReadFile("providers/" + provider + "/go.mod")
goModContent, err := os.ReadFile(goModPath)
if err != nil {
log.Fatal().Err(err).Str("provider", provider).Msg("failed to read provider go.mod")
log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to read provider go.mod")
}
goMod, err := modfile.Parse(fmt.Sprintf("%s/go.mod", provider), goModContent, nil)
goMod, err := modfile.Parse(goModPath, goModContent, nil)
if err != nil {
log.Fatal().Err(err).Str("provider", provider).Msg("failed to parse provider go.mod")
log.Fatal().Err(err).Str("provider", provider.Name).Msg("failed to parse provider go.mod")
}

// we don't care about the specific version for dev
deps += "\n\tgo.mondoo.com/cnquery/v11/providers/" + provider + " v0.0.0"
replace += "\nreplace go.mondoo.com/cnquery/v11/providers/" + provider + " => ./providers/" + provider
deps += "\n\t" + provider.GoPackage + " v0.0.0"
if provider.Remote != "" {
replace += "\nreplace " + provider.GoPackage + " => " + provider.Remote
} else {
replace += "\nreplace " + provider.GoPackage + " => ./providers/" + provider.Name
}
// if the provider has any specific pinned replacements, we also add those to allow compiling
for _, r := range goMod.Replace {
// special case: we don't want to pull in provider's 'replace go.mondoo.com/cnquery/v11 => ../..' in.
Expand Down
102 changes: 102 additions & 0 deletions providers-sdk/v1/util/configure/provider_conf.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
// Copyright (c) Mondoo, Inc.
// SPDX-License-Identifier: BUSL-1.1
package main

import (
"encoding/json"
"fmt"
"os"
"strings"
)

type ProvidersConf struct {
Builtin []Builtin `json:"builtin"`

providers []string // Providers names
}

func (c ProvidersConf) Providers() []string {
if len(c.providers) == 0 {
for _, b := range c.Builtin {
c.providers = append(c.providers, b.Name)
}
}
return c.providers
}

type Builtin struct {
Name string
Remote string
GoPackage string
}

func (b Builtin) Cwd() string {
if b.Remote != "" {
return b.Remote
}

return ""
}

func (b Builtin) Resource() string {
if b.Remote != "" {
return fmt.Sprintf("%s/resources/%s.resources.json", b.Remote, b.Name)
}
return "providers/" + b.Name + "/resources/" + b.Name + ".resources.json"
}

func (b Builtin) Dist() string {
return "providers/" + b.Name + ".resources.json"
}

func (b Builtin) GoMod() string {
if b.Remote != "" {
return b.Remote + "/go.mod"
}
return "providers/" + b.Name + "/go.mod"
}

func (b *Builtin) UnmarshalJSON(data []byte) error {
var name string
if err := json.Unmarshal(data, &name); err == nil {
b.Name = name
b.GoPackage = "go.mondoo.com/cnquery/v11/providers/" + name

return nil
}

var raw struct {
Name string `json:"name"`
Remote string `json:"remote"`
GoPackage string `json:"goPackage"`
}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}

b.Name = raw.Name
b.Remote = raw.Remote
if strings.HasPrefix(b.Remote, "~/") {
b.Remote = os.ExpandEnv(strings.Replace(b.Remote, "~/", "$HOME/", 1))
}

b.GoPackage = raw.GoPackage

return nil
}

func (b *Builtin) MarshalJSON() ([]byte, error) {
if b.Remote == "" && b.GoPackage == "" {
return json.Marshal(b.Name)
}

return json.Marshal(struct {
Name string `json:"name"`
Remote string `json:"remote"`
GoPackage string `json:"goPackage"`
}{
Name: b.Name,
Remote: b.Remote,
GoPackage: b.GoPackage,
})
}
7 changes: 7 additions & 0 deletions providers.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,11 @@
# In this case it will be pulled from your local repo
# instead of using the pre-installed provider in
# your OS (like ~/.mondoo/providers/os).
#
# If you want to test a provider that is not part of
# the cnquery repo, you can add it here as well.
# Example:
# - name: my-awesome-provider
# remote: ~/path/to/my-awesome-provider
# goPackage: github.com/me/my-awesome-provider
builtin: []
2 changes: 1 addition & 1 deletion providers/builtin_dev.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// SPDX-License-Identifier: BUSL-1.1
//
// This file is auto-generated by 'make providers/config'
// and configured via 'providers.yaml'
// and configured via 'providers.yaml'; DO NOT EDIT.

package providers

Expand Down

0 comments on commit 2d94402

Please sign in to comment.