From 5fc34c8499fc9a471da4f9c301addac9b5a5f953 Mon Sep 17 00:00:00 2001 From: Christoph Hartmann Date: Mon, 15 Jan 2024 19:36:20 +0100 Subject: [PATCH] =?UTF-8?q?=E2=AD=90=EF=B8=8F=20support=20custom=20provide?= =?UTF-8?q?r=20path=20(#3031)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧹 update provider reference * ⭐️ support custom provider path --- .github/workflows/pr-test-lint.yml | 9 ++++-- apps/cnquery/cmd/providers.go | 9 ++++-- providers/arista/go.mod | 2 +- providers/atlassian/go.mod | 2 +- providers/aws/go.mod | 2 +- providers/azure/go.mod | 2 +- providers/equinix/go.mod | 2 +- providers/gcp/go.mod | 2 +- providers/github/go.mod | 2 +- providers/gitlab/go.mod | 2 +- providers/google-workspace/go.mod | 2 +- providers/ipmi/go.mod | 2 +- providers/k8s/go.mod | 2 +- providers/ms365/go.mod | 2 +- providers/oci/go.mod | 2 +- providers/okta/go.mod | 2 +- providers/opcua/go.mod | 2 +- providers/providers.go | 25 ++++++++++++---- providers/slack/go.mod | 2 +- providers/terraform/go.mod | 2 +- providers/vcd/go.mod | 2 +- providers/vsphere/go.mod | 2 +- test/cli/testdata/cnquery_run.ct | 47 +++++------------------------- test/cli/testdata/cnquery_sbom.ct | 16 ---------- test/cli/testdata/cnquery_scan.ct | 33 +-------------------- 25 files changed, 62 insertions(+), 115 deletions(-) diff --git a/.github/workflows/pr-test-lint.yml b/.github/workflows/pr-test-lint.yml index c144c4d2bc..88c618d72f 100644 --- a/.github/workflows/pr-test-lint.yml +++ b/.github/workflows/pr-test-lint.yml @@ -84,15 +84,20 @@ jobs: - name: 'Set up gcloud CLI' uses: 'google-github-actions/setup-gcloud@v2' + - name: Set provider env + run: echo "PROVIDERS_PATH=${PWD}/.providers" >> $GITHUB_ENV + - name: Display Provider PAth + run: echo $PROVIDERS_PATH + - name: Test cnquery run: make test/go/plain-ci - + - name: Test Providers run: make providers/test - uses: actions/upload-artifact@v4 # upload test results if: success() || failure() # run this step even if previous step failed - with: + with: name: test-results path: report.xml diff --git a/apps/cnquery/cmd/providers.go b/apps/cnquery/cmd/providers.go index 04fbb4847f..fb1e05a7b6 100644 --- a/apps/cnquery/cmd/providers.go +++ b/apps/cnquery/cmd/providers.go @@ -160,11 +160,16 @@ func printProviders(p []*providers.Provider) { } printProviderPath("builtin", paths["builtin"], false) - printProviderPath(providers.HomePath, paths[providers.HomePath], true) - printProviderPath(providers.SystemPath, paths[providers.SystemPath], true) + if providers.CustomProviderPath == "" { + printProviderPath(providers.HomePath, paths[providers.HomePath], true) + printProviderPath(providers.SystemPath, paths[providers.SystemPath], true) + } else { + printProviderPath(providers.CustomProviderPath, paths[providers.CustomProviderPath], true) + } delete(paths, "builtin") delete(paths, providers.HomePath) delete(paths, providers.SystemPath) + delete(paths, providers.CustomProviderPath) keys := sortx.Keys(paths) for _, path := range keys { diff --git a/providers/arista/go.mod b/providers/arista/go.mod index 11f53e14c5..799b988df4 100644 --- a/providers/arista/go.mod +++ b/providers/arista/go.mod @@ -9,7 +9,7 @@ toolchain go1.21.3 require ( github.com/aristanetworks/goeapi v1.0.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/atlassian/go.mod b/providers/atlassian/go.mod index 4568529e09..9d4b5a1e05 100644 --- a/providers/atlassian/go.mod +++ b/providers/atlassian/go.mod @@ -9,7 +9,7 @@ toolchain go1.21.3 require ( github.com/ctreminiom/go-atlassian v1.5.1 github.com/pkg/errors v0.9.1 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/aws/go.mod b/providers/aws/go.mod index 7f512dcecd..38c6fd2312 100644 --- a/providers/aws/go.mod +++ b/providers/aws/go.mod @@ -59,7 +59,7 @@ require ( github.com/rs/zerolog v1.31.0 github.com/spf13/afero v1.11.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 k8s.io/client-go v0.29.0 ) diff --git a/providers/azure/go.mod b/providers/azure/go.mod index c6bd1a1fb2..1d330ad0c2 100644 --- a/providers/azure/go.mod +++ b/providers/azure/go.mod @@ -36,7 +36,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/equinix/go.mod b/providers/equinix/go.mod index 57772584dd..e03f0affa9 100644 --- a/providers/equinix/go.mod +++ b/providers/equinix/go.mod @@ -9,7 +9,7 @@ toolchain go1.21.3 require ( github.com/packethost/packngo v0.31.0 github.com/rs/zerolog v1.31.0 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/gcp/go.mod b/providers/gcp/go.mod index 9c242686de..5aeab0fa2f 100644 --- a/providers/gcp/go.mod +++ b/providers/gcp/go.mod @@ -27,7 +27,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 go.mondoo.com/ranger-rpc v0.5.3 golang.org/x/oauth2 v0.16.0 google.golang.org/api v0.156.0 diff --git a/providers/github/go.mod b/providers/github/go.mod index c833d58cf7..64a3a7e669 100644 --- a/providers/github/go.mod +++ b/providers/github/go.mod @@ -11,7 +11,7 @@ require ( github.com/google/go-github/v57 v57.0.0 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 go.mondoo.com/ranger-rpc v0.5.3 golang.org/x/oauth2 v0.16.0 sigs.k8s.io/yaml v1.4.0 diff --git a/providers/gitlab/go.mod b/providers/gitlab/go.mod index 9539b9fc3e..d75939bcaa 100644 --- a/providers/gitlab/go.mod +++ b/providers/gitlab/go.mod @@ -8,7 +8,7 @@ require ( github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 github.com/xanzy/go-gitlab v0.95.2 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 google.golang.org/protobuf v1.32.0 ) diff --git a/providers/google-workspace/go.mod b/providers/google-workspace/go.mod index 36d9c3c252..9e43279aa9 100644 --- a/providers/google-workspace/go.mod +++ b/providers/google-workspace/go.mod @@ -8,7 +8,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 golang.org/x/oauth2 v0.16.0 google.golang.org/api v0.156.0 ) diff --git a/providers/ipmi/go.mod b/providers/ipmi/go.mod index 63a009818a..2dd80e98bf 100644 --- a/providers/ipmi/go.mod +++ b/providers/ipmi/go.mod @@ -10,7 +10,7 @@ require ( github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 github.com/vmware/goipmi v0.0.0-20181114221114-2333cd82d702 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/k8s/go.mod b/providers/k8s/go.mod index e986d40c34..4606018f3e 100644 --- a/providers/k8s/go.mod +++ b/providers/k8s/go.mod @@ -14,7 +14,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 // pin v0.28.4 k8s.io/api v0.28.4 // pin v0.28.4 diff --git a/providers/ms365/go.mod b/providers/ms365/go.mod index c3d4cc7a07..711951a1c4 100644 --- a/providers/ms365/go.mod +++ b/providers/ms365/go.mod @@ -15,7 +15,7 @@ require ( github.com/microsoftgraph/msgraph-sdk-go v1.29.0 github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/oci/go.mod b/providers/oci/go.mod index 8dd2e93d23..8a5a086e35 100644 --- a/providers/oci/go.mod +++ b/providers/oci/go.mod @@ -9,7 +9,7 @@ replace go.mondoo.com/cnquery/v10 => ../.. require ( github.com/oracle/oci-go-sdk/v65 v65.55.1 github.com/rs/zerolog v1.31.0 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/okta/go.mod b/providers/okta/go.mod index 8bf743fbd9..7a96ee2969 100644 --- a/providers/okta/go.mod +++ b/providers/okta/go.mod @@ -10,7 +10,7 @@ require ( github.com/okta/okta-sdk-golang/v2 v2.20.0 github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 go.mondoo.com/ranger-rpc v0.5.3 ) diff --git a/providers/opcua/go.mod b/providers/opcua/go.mod index d6f93de8cf..6e374da46e 100644 --- a/providers/opcua/go.mod +++ b/providers/opcua/go.mod @@ -9,7 +9,7 @@ toolchain go1.21.3 require ( github.com/gopcua/opcua v0.5.3 github.com/mozillazg/go-slugify v0.2.0 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/providers.go b/providers/providers.go index fab457dc90..fbeaf7a68d 100644 --- a/providers/providers.go +++ b/providers/providers.go @@ -29,8 +29,9 @@ import ( ) var ( - SystemPath string - HomePath string + SystemPath string + HomePath string + CustomProviderPath string // this is the default path for providers, it's either system or home path, if the user is root the system path is used DefaultPath string // CachedProviders contains all providers that have been loaded the last time @@ -49,6 +50,10 @@ func init() { HomePath, _ = config.HomePath("providers") DefaultPath = HomePath } + CustomProviderPath = os.Getenv("PROVIDERS_PATH") + if CustomProviderPath != "" { + DefaultPath = CustomProviderPath + } LastProviderInstall = time.Now().Unix() } @@ -186,7 +191,7 @@ func ListAll() ([]*Provider, error) { CachedProviders = all // This really shouldn't happen, but just in case it does... - if SystemPath == "" && HomePath == "" { + if SystemPath == "" && HomePath == "" && CustomProviderPath == "" { log.Warn().Msg("can't find any paths for providers, none are configured") return nil, nil } @@ -204,7 +209,17 @@ func ListAll() ([]*Provider, error) { msg.Msg("can't find any paths for providers, none are configured") } - if sysOk { + // when the user provides a custom provider path, we always load it and we ignore the system and home path + // we do not check for its existence, and instead create it on the fly when needed + if CustomProviderPath != "" { + cur, err := findProviders(CustomProviderPath) + if err != nil { + log.Warn().Str("path", CustomProviderPath).Err(err).Msg("failed to get providers from custom provider path") + } + all = append(all, cur...) + } + + if sysOk && CustomProviderPath == "" { cur, err := findProviders(SystemPath) if err != nil { log.Warn().Str("path", SystemPath).Err(err).Msg("failed to get providers from system path") @@ -212,7 +227,7 @@ func ListAll() ([]*Provider, error) { all = append(all, cur...) } - if homeOk { + if homeOk && CustomProviderPath == "" { cur, err := findProviders(HomePath) if err != nil { log.Warn().Str("path", HomePath).Err(err).Msg("failed to get providers from home path") diff --git a/providers/slack/go.mod b/providers/slack/go.mod index 0040ad9a75..066367d147 100644 --- a/providers/slack/go.mod +++ b/providers/slack/go.mod @@ -10,7 +10,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 github.com/rs/zerolog v1.31.0 github.com/slack-go/slack v0.12.3 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/terraform/go.mod b/providers/terraform/go.mod index 3d080b6ead..72ff529796 100644 --- a/providers/terraform/go.mod +++ b/providers/terraform/go.mod @@ -9,7 +9,7 @@ toolchain go1.21.3 require ( github.com/go-git/go-git/v5 v5.11.0 github.com/hashicorp/hcl/v2 v2.19.1 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/vcd/go.mod b/providers/vcd/go.mod index 0034925c5c..544cfdd4bb 100644 --- a/providers/vcd/go.mod +++ b/providers/vcd/go.mod @@ -10,7 +10,7 @@ require ( github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 github.com/vmware/go-vcloud-director/v2 v2.22.0 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 ) require ( diff --git a/providers/vsphere/go.mod b/providers/vsphere/go.mod index 11d7e734c1..e0ed74aca9 100644 --- a/providers/vsphere/go.mod +++ b/providers/vsphere/go.mod @@ -11,7 +11,7 @@ require ( github.com/rs/zerolog v1.31.0 github.com/stretchr/testify v1.8.4 github.com/vmware/govmomi v0.34.2 - go.mondoo.com/cnquery/v10 v10.0.0 + go.mondoo.com/cnquery/v10 v10.0.0-beta1 go.mondoo.com/mondoo-go v0.0.0-20240103050411-fa7e920c6784 ) diff --git a/test/cli/testdata/cnquery_run.ct b/test/cli/testdata/cnquery_run.ct index 18acbf1adf..17448f51ff 100644 --- a/test/cli/testdata/cnquery_run.ct +++ b/test/cli/testdata/cnquery_run.ct @@ -6,47 +6,16 @@ Usage: cnquery run [command] Available Commands: - arista Run a query with an Arista EOS device - atlassian Run a query with Atlassian - aws Run a query with an AWS account - azure Run a query with an Azure subscription - container Run a query with a running container or container image - docker Run a query with a running Docker container or Docker image - equinix Run a query with an Equinix Metal organization - filesystem Run a query with a mounted file system target - gcp Run a query with a Google Cloud project - github Run a query with a GitHub organization or repository - gitlab Run a query with a GitLab group or project - google-workspace Run a query with a Google Workspace account - host Run a query with a remote host - ipmi Run a query with an IPMI interface - k8s Run a query with a Kubernetes cluster or local manifest file(s) - local Run a query with your local system - mock Run a query with use a recording without an active connection - ms365 Run a query with a Microsoft 365 account - oci Run a query with an Oracle Cloud Infrastructure tenancy - okta Run a query with Okta - opcua Run a query with an OPC UA device - slack Run a query with a Slack team - ssh Run a query with a remote system via SSH - terraform Run a query with a Terraform HCL file or directory - vagrant Run a query with a Vagrant host - vcd Run a query with a VMware Cloud Director installation - vsphere Run a query with a VMware vSphere installation - winrm Run a query with a remote system via WinRM + mock Run a query with use a recording without an active connection Flags: - --ast Parse the query and return the abstract syntax tree (AST). - -c, --command string MQL query to executed in the shell. - --discover strings Enable the discovery of nested assets. Supports: all,auto,container,container-images - -h, --help help for run - --info Parse the query and provide information about it. - -j, --json Run the query and return the object in a JSON structure. - --parse Parse the query and return the logical structure. - --platform-id string Select a specific target asset by providing its platform ID. - --record string Record all resource calls and use resources in the recording - --sudo Elevate privileges with sudo. - --use-recording string Use a recording to inject resource data (read-only) + --ast Parse the query and return the abstract syntax tree (AST). + -c, --command string MQL query to executed in the shell. + -h, --help help for run + --info Parse the query and provide information about it. + -j, --json Run the query and return the object in a JSON structure. + --parse Parse the query and return the logical structure. + --platform-id string Select a specific target asset by providing its platform ID. Global Flags: --api-proxy string Set proxy for communications with Mondoo API diff --git a/test/cli/testdata/cnquery_sbom.ct b/test/cli/testdata/cnquery_sbom.ct index 88465e2ebe..171451ebc7 100644 --- a/test/cli/testdata/cnquery_sbom.ct +++ b/test/cli/testdata/cnquery_sbom.ct @@ -14,26 +14,12 @@ Note this command is experimental and may change in the future. Usage: cnquery sbom [flags] - cnquery sbom [command] - -Available Commands: - container Collect a software bill of materials (SBOM) for a running container or container image - docker Collect a software bill of materials (SBOM) for a running Docker container or Docker image - filesystem Collect a software bill of materials (SBOM) for a mounted file system target - local Collect a software bill of materials (SBOM) for your local system - ssh Collect a software bill of materials (SBOM) for a remote system via SSH - vagrant Collect a software bill of materials (SBOM) for a Vagrant host - winrm Collect a software bill of materials (SBOM) for a remote system via WinRM Flags: --annotation stringToString Add an annotation to the asset. (default []) --asset-name string User-override for the asset name - --discover strings Enable the discovery of nested assets. Supports: all,auto,container,container-images -h, --help help for sbom -o, --output string Set output format: json, cyclonedx-json, cyclonedx-xml, spdx-json, spdx-tag-value, table (default "list") - --record string Record all resource calls and use resources in the recording - --sudo Elevate privileges with sudo. - --use-recording string Use a recording to inject resource data (read-only) Global Flags: --api-proxy string Set proxy for communications with Mondoo API @@ -41,5 +27,3 @@ Global Flags: --config string Set config file path (default $HOME/.config/mondoo/mondoo.yml) --log-level string Set log level: error, warn, info, debug, trace (default "info") -v, --verbose Enable verbose output - -Use "cnquery sbom [command] --help" for more information about a command. diff --git a/test/cli/testdata/cnquery_scan.ct b/test/cli/testdata/cnquery_scan.ct index c18c5abb69..fcd30e3e51 100644 --- a/test/cli/testdata/cnquery_scan.ct +++ b/test/cli/testdata/cnquery_scan.ct @@ -14,40 +14,12 @@ Usage: cnquery scan [command] Available Commands: - arista Scan an Arista EOS device - atlassian Scan Atlassian - aws Scan an AWS account - azure Scan an Azure subscription - container Scan a running container or container image - docker Scan a running Docker container or Docker image - equinix Scan an Equinix Metal organization - filesystem Scan a mounted file system target - gcp Scan a Google Cloud project - github Scan a GitHub organization or repository - gitlab Scan a GitLab group or project - google-workspace Scan a Google Workspace account - host Scan a remote host - ipmi Scan an IPMI interface - k8s Scan a Kubernetes cluster or local manifest file(s) - local Scan your local system - mock Scan use a recording without an active connection - ms365 Scan a Microsoft 365 account - oci Scan an Oracle Cloud Infrastructure tenancy - okta Scan Okta - opcua Scan an OPC UA device - slack Scan a Slack team - ssh Scan a remote system via SSH - terraform Scan a Terraform HCL file or directory - vagrant Scan a Vagrant host - vcd Scan a VMware Cloud Director installation - vsphere Scan a VMware vSphere installation - winrm Scan a remote system via WinRM + mock Scan use a recording without an active connection Flags: --annotation stringToString Add an annotation to the asset. (default []) --asset-name string User-override for the asset name --detect-cicd Try to detect CI/CD environments. If detected, set the asset category to 'cicd'. (default true) - --discover strings Enable the discovery of nested assets. Supports: all,auto,container,container-images -h, --help help for scan --incognito Run in incognito mode. Do not report scan results to Mondoo Platform. --inventory-ansible Set the inventory format to Ansible. @@ -59,9 +31,6 @@ Flags: --props stringToString Custom values for properties (default []) --querypack querypack-bundle Set the query packs to execute. This requires querypack-bundle. You can specify multiple UIDs. -f, --querypack-bundle strings Path to local query pack file - --record string Record all resource calls and use resources in the recording - --sudo Elevate privileges with sudo. - --use-recording string Use a recording to inject resource data (read-only) Global Flags: --api-proxy string Set proxy for communications with Mondoo API