⭐️ support custom provider path (#3031)

* 🧹 update provider reference * ⭐️ support custom provider path
mondoohq · Jan 15, 2024 · 5fc34c8 · 5fc34c8
1 parent 3c1bfad
commit 5fc34c8
Show file tree

Hide file tree

Showing 25 changed files with 62 additions and 115 deletions.
diff --git a/.github/workflows/pr-test-lint.yml b/.github/workflows/pr-test-lint.yml
@@ -84,15 +84,20 @@ jobs:
       - name: 'Set up gcloud CLI'
         uses: 'google-github-actions/setup-gcloud@v2'
 
+      - name: Set provider env
+        run: echo "PROVIDERS_PATH=${PWD}/.providers" >> $GITHUB_ENV
+      - name: Display Provider PAth
+        run: echo $PROVIDERS_PATH
+
       - name: Test cnquery
         run: make test/go/plain-ci
-        
+
       - name: Test Providers
         run: make providers/test
 
       - uses: actions/upload-artifact@v4  # upload test results
         if: success() || failure()        # run this step even if previous step failed
-        with:                           
+        with:
           name: test-results
           path: report.xml
 

diff --git a/apps/cnquery/cmd/providers.go b/apps/cnquery/cmd/providers.go
@@ -160,11 +160,16 @@ func printProviders(p []*providers.Provider) {
 	}
 
 	printProviderPath("builtin", paths["builtin"], false)
-	printProviderPath(providers.HomePath, paths[providers.HomePath], true)
-	printProviderPath(providers.SystemPath, paths[providers.SystemPath], true)
+	if providers.CustomProviderPath == "" {
+		printProviderPath(providers.HomePath, paths[providers.HomePath], true)
+		printProviderPath(providers.SystemPath, paths[providers.SystemPath], true)
+	} else {
+		printProviderPath(providers.CustomProviderPath, paths[providers.CustomProviderPath], true)
+	}
 	delete(paths, "builtin")
 	delete(paths, providers.HomePath)
 	delete(paths, providers.SystemPath)
+	delete(paths, providers.CustomProviderPath)
 
 	keys := sortx.Keys(paths)
 	for _, path := range keys {

diff --git a/providers/arista/go.mod b/providers/arista/go.mod
@@ -9,7 +9,7 @@ toolchain go1.21.3
 require (
 	github.com/aristanetworks/goeapi v1.0.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/atlassian/go.mod b/providers/atlassian/go.mod
@@ -9,7 +9,7 @@ toolchain go1.21.3
 require (
 	github.com/ctreminiom/go-atlassian v1.5.1
 	github.com/pkg/errors v0.9.1
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/aws/go.mod b/providers/aws/go.mod
@@ -59,7 +59,7 @@ require (
 	github.com/rs/zerolog v1.31.0
 	github.com/spf13/afero v1.11.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	k8s.io/client-go v0.29.0
 )
 

diff --git a/providers/azure/go.mod b/providers/azure/go.mod
@@ -36,7 +36,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/equinix/go.mod b/providers/equinix/go.mod
@@ -9,7 +9,7 @@ toolchain go1.21.3
 require (
 	github.com/packethost/packngo v0.31.0
 	github.com/rs/zerolog v1.31.0
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/gcp/go.mod b/providers/gcp/go.mod
@@ -27,7 +27,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	go.mondoo.com/ranger-rpc v0.5.3
 	golang.org/x/oauth2 v0.16.0
 	google.golang.org/api v0.156.0

diff --git a/providers/github/go.mod b/providers/github/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/google/go-github/v57 v57.0.0
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	go.mondoo.com/ranger-rpc v0.5.3
 	golang.org/x/oauth2 v0.16.0
 	sigs.k8s.io/yaml v1.4.0

diff --git a/providers/gitlab/go.mod b/providers/gitlab/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
 	github.com/xanzy/go-gitlab v0.95.2
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
 	google.golang.org/protobuf v1.32.0
 )

diff --git a/providers/google-workspace/go.mod b/providers/google-workspace/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	golang.org/x/oauth2 v0.16.0
 	google.golang.org/api v0.156.0
 )

diff --git a/providers/ipmi/go.mod b/providers/ipmi/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
 	github.com/vmware/goipmi v0.0.0-20181114221114-2333cd82d702
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/k8s/go.mod b/providers/k8s/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	// pin v0.28.4
 	k8s.io/api v0.28.4
 	// pin v0.28.4

diff --git a/providers/ms365/go.mod b/providers/ms365/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/microsoftgraph/msgraph-sdk-go v1.29.0
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.31.0
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/oci/go.mod b/providers/oci/go.mod
@@ -9,7 +9,7 @@ replace go.mondoo.com/cnquery/v10 => ../..
 require (
 	github.com/oracle/oci-go-sdk/v65 v65.55.1
 	github.com/rs/zerolog v1.31.0
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/okta/go.mod b/providers/okta/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/okta/okta-sdk-golang/v2 v2.20.0
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	go.mondoo.com/ranger-rpc v0.5.3
 )
 

diff --git a/providers/opcua/go.mod b/providers/opcua/go.mod
@@ -9,7 +9,7 @@ toolchain go1.21.3
 require (
 	github.com/gopcua/opcua v0.5.3
 	github.com/mozillazg/go-slugify v0.2.0
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/providers.go b/providers/providers.go
@@ -29,8 +29,9 @@ import (
 )
 
 var (
-	SystemPath string
-	HomePath   string
+	SystemPath         string
+	HomePath           string
+	CustomProviderPath string
 	// this is the default path for providers, it's either system or home path, if the user is root the system path is used
 	DefaultPath string
 	// CachedProviders contains all providers that have been loaded the last time
@@ -49,6 +50,10 @@ func init() {
 		HomePath, _ = config.HomePath("providers")
 		DefaultPath = HomePath
 	}
+	CustomProviderPath = os.Getenv("PROVIDERS_PATH")
+	if CustomProviderPath != "" {
+		DefaultPath = CustomProviderPath
+	}
 
 	LastProviderInstall = time.Now().Unix()
 }
@@ -186,7 +191,7 @@ func ListAll() ([]*Provider, error) {
 	CachedProviders = all
 
 	// This really shouldn't happen, but just in case it does...
-	if SystemPath == "" && HomePath == "" {
+	if SystemPath == "" && HomePath == "" && CustomProviderPath == "" {
 		log.Warn().Msg("can't find any paths for providers, none are configured")
 		return nil, nil
 	}
@@ -204,15 +209,25 @@ func ListAll() ([]*Provider, error) {
 		msg.Msg("can't find any paths for providers, none are configured")
 	}
 
-	if sysOk {
+	// when the user provides a custom provider path, we always load it and we ignore the system and home path
+	// we do not check for its existence, and instead create it on the fly when needed
+	if CustomProviderPath != "" {
+		cur, err := findProviders(CustomProviderPath)
+		if err != nil {
+			log.Warn().Str("path", CustomProviderPath).Err(err).Msg("failed to get providers from custom provider path")
+		}
+		all = append(all, cur...)
+	}
+
+	if sysOk && CustomProviderPath == "" {
 		cur, err := findProviders(SystemPath)
 		if err != nil {
 			log.Warn().Str("path", SystemPath).Err(err).Msg("failed to get providers from system path")
 		}
 		all = append(all, cur...)
 	}
 
-	if homeOk {
+	if homeOk && CustomProviderPath == "" {
 		cur, err := findProviders(HomePath)
 		if err != nil {
 			log.Warn().Str("path", HomePath).Err(err).Msg("failed to get providers from home path")

diff --git a/providers/slack/go.mod b/providers/slack/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.7.5
 	github.com/rs/zerolog v1.31.0
 	github.com/slack-go/slack v0.12.3
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/terraform/go.mod b/providers/terraform/go.mod
@@ -9,7 +9,7 @@ toolchain go1.21.3
 require (
 	github.com/go-git/go-git/v5 v5.11.0
 	github.com/hashicorp/hcl/v2 v2.19.1
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/vcd/go.mod b/providers/vcd/go.mod
@@ -10,7 +10,7 @@ require (
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
 	github.com/vmware/go-vcloud-director/v2 v2.22.0
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 )
 
 require (

diff --git a/providers/vsphere/go.mod b/providers/vsphere/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/rs/zerolog v1.31.0
 	github.com/stretchr/testify v1.8.4
 	github.com/vmware/govmomi v0.34.2
-	go.mondoo.com/cnquery/v10 v10.0.0
+	go.mondoo.com/cnquery/v10 v10.0.0-beta1
 	go.mondoo.com/mondoo-go v0.0.0-20240103050411-fa7e920c6784
 )
 

diff --git a/test/cli/testdata/cnquery_run.ct b/test/cli/testdata/cnquery_run.ct
@@ -6,47 +6,16 @@ Usage:
   cnquery run [command]
 
 Available Commands:
-  arista           Run a query with an Arista EOS device
-  atlassian        Run a query with Atlassian
-  aws              Run a query with an AWS account
-  azure            Run a query with an Azure subscription
-  container        Run a query with a running container or container image
-  docker           Run a query with a running Docker container or Docker image
-  equinix          Run a query with an Equinix Metal organization
-  filesystem       Run a query with a mounted file system target
-  gcp              Run a query with a Google Cloud project
-  github           Run a query with a GitHub organization or repository
-  gitlab           Run a query with a GitLab group or project
-  google-workspace Run a query with a Google Workspace account
-  host             Run a query with a remote host
-  ipmi             Run a query with an IPMI interface
-  k8s              Run a query with a Kubernetes cluster or local manifest file(s)
-  local            Run a query with your local system
-  mock             Run a query with use a recording without an active connection
-  ms365            Run a query with a Microsoft 365 account
-  oci              Run a query with an Oracle Cloud Infrastructure tenancy
-  okta             Run a query with Okta
-  opcua            Run a query with an OPC UA device
-  slack            Run a query with a Slack team
-  ssh              Run a query with a remote system via SSH
-  terraform        Run a query with a Terraform HCL file or directory
-  vagrant          Run a query with a Vagrant host
-  vcd              Run a query with a VMware Cloud Director installation
-  vsphere          Run a query with a VMware vSphere installation
-  winrm            Run a query with a remote system via WinRM
+  mock        Run a query with use a recording without an active connection
 
 Flags:
-      --ast                    Parse the query and return the abstract syntax tree (AST).
-  -c, --command string         MQL query to executed in the shell.
-      --discover strings       Enable the discovery of nested assets. Supports: all,auto,container,container-images
-  -h, --help                   help for run
-      --info                   Parse the query and provide information about it.
-  -j, --json                   Run the query and return the object in a JSON structure.
-      --parse                  Parse the query and return the logical structure.
-      --platform-id string     Select a specific target asset by providing its platform ID.
-      --record string          Record all resource calls and use resources in the recording
-      --sudo                   Elevate privileges with sudo.
-      --use-recording string   Use a recording to inject resource data (read-only)
+      --ast                  Parse the query and return the abstract syntax tree (AST).
+  -c, --command string       MQL query to executed in the shell.
+  -h, --help                 help for run
+      --info                 Parse the query and provide information about it.
+  -j, --json                 Run the query and return the object in a JSON structure.
+      --parse                Parse the query and return the logical structure.
+      --platform-id string   Select a specific target asset by providing its platform ID.
 
 Global Flags:
       --api-proxy string   Set proxy for communications with Mondoo API

diff --git a/test/cli/testdata/cnquery_sbom.ct b/test/cli/testdata/cnquery_sbom.ct
@@ -14,32 +14,16 @@ Note this command is experimental and may change in the future.
 
 Usage:
   cnquery sbom [flags]
-  cnquery sbom [command]
-
-Available Commands:
-  container   Collect a software bill of materials (SBOM) for a running container or container image
-  docker      Collect a software bill of materials (SBOM) for a running Docker container or Docker image
-  filesystem  Collect a software bill of materials (SBOM) for a mounted file system target
-  local       Collect a software bill of materials (SBOM) for your local system
-  ssh         Collect a software bill of materials (SBOM) for a remote system via SSH
-  vagrant     Collect a software bill of materials (SBOM) for a Vagrant host
-  winrm       Collect a software bill of materials (SBOM) for a remote system via WinRM
 
 Flags:
       --annotation stringToString   Add an annotation to the asset. (default [])
       --asset-name string           User-override for the asset name
-      --discover strings            Enable the discovery of nested assets. Supports: all,auto,container,container-images
   -h, --help                        help for sbom
   -o, --output string               Set output format: json, cyclonedx-json, cyclonedx-xml, spdx-json, spdx-tag-value, table (default "list")
-      --record string               Record all resource calls and use resources in the recording
-      --sudo                        Elevate privileges with sudo.
-      --use-recording string        Use a recording to inject resource data (read-only)
 
 Global Flags:
       --api-proxy string   Set proxy for communications with Mondoo API
       --auto-update        Enable automatic provider installation and update (default true)
       --config string      Set config file path (default $HOME/.config/mondoo/mondoo.yml)
       --log-level string   Set log level: error, warn, info, debug, trace (default "info")
   -v, --verbose            Enable verbose output
-
-Use "cnquery sbom [command] --help" for more information about a command.