Merge branch 'main' into afiune/validate-gh-conn-once

.github/actions/spelling/expect.txt

@@ -48,6 +48,7 @@ ipsetreferencestatement
 istio
 jira
 jsonbody
+kqueue
 labelmatchstatement
 liveanalytics
 loggingservice
@@ -60,13 +61,15 @@ mgroup
 Mpim
 natgateway
 networkinterface
+nmap
 nodegroup
 nodepool
 notebookinstancedetails
 nsrecord
 nullgroup
 nullstring
 opcplc
+openssl
 orstatement
 PAYG
 Pids
@@ -80,6 +83,7 @@ regexpatternsetreferencestatement
 resourcegroup
 rulegroup
 rulegroupreferencestatement
+rtsp
 Sas
 scim
 serviceprincipals
@@ -100,7 +104,6 @@ timestream
 toplevel
 tpu
 vdcs
-VNIC
 Vtpm
 vulnerabilityassessmentsettings
 vulnmgmt

.github/workflows/main-benchmark.yml

@@ -63,4 +63,4 @@ jobs:
         uses: actions/cache/save@v4
         with:
           path: ./cache
-          key: ${{ runner.os }}-benchmark-${{ github.run_id }} 
+          key: ${{ runner.os }}-benchmark-${{ github.run_id }}

.github/workflows/pr-test-lint.yml

@@ -128,6 +128,24 @@ jobs:
           name: test-results-cli
           path: report.xml
 
+  go-race:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Import environment variables from file
+        run: cat ".github/env" >> $GITHUB_ENV
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ">=${{ env.golang-version }}"
+          cache: false
+
+      - name: Run race detector on selected packages
+        run: make race/go
+
   go-bench:
     runs-on: ubuntu-latest
     if: github.ref != 'refs/heads/main'

.vscode/launch.json

@@ -198,6 +198,18 @@
         "shell", "ssh", "[email protected]",
       ],
     },
+    {
+      "name": "scan github org",
+      "type": "go",
+      "request": "launch",
+      "program": "${workspaceRoot}/apps/cnquery/cnquery.go",
+      "args": [
+        "scan",
+        "github",
+        "org", "hit-training",
+        "--log-level", "trace"
+      ]
+    },
     {
       "name": "Configure Built-in Providers",
       "type": "go",

Makefile

@@ -213,7 +213,8 @@ providers/build: \
 	providers/build/ansible \
 	providers/build/snowflake \
 	providers/build/mondoo \
-	providers/build/cloudflare
+	providers/build/cloudflare \
+	providers/build/nmap
 
 .PHONY: providers/install
 # Note we need \ to escape the target line into multiple lines
@@ -244,7 +245,8 @@ providers/install: \
 	providers/install/ansible \
 	providers/install/snowflake \
 	providers/install/mondoo \
-	providers/install/cloudflare
+	providers/install/cloudflare \
+	providers/build/nmap
 
 providers/build/mock: providers/lr
 	./lr go providers-sdk/v1/testutils/mockprovider/resources/mockprovider.lr
@@ -387,6 +389,11 @@ providers/build/mondoo: providers/lr
 providers/install/mondoo:
 	@$(call installProvider, providers/mondoo)
 
+providers/build/nmap: providers/lr
+	@$(call buildProvider, providers/nmap)
+providers/install/nmap:
+	@$(call installProvider, providers/nmap)
+
 providers/dist:
 	@$(call buildProviderDist, providers/network)
 	@$(call buildProviderDist, providers/os)
@@ -414,6 +421,7 @@ providers/dist:
 	@$(call buildProviderDist, providers/ansible)
 	@$(call buildProviderDist, providers/snowflake)
 	@$(call buildProviderDist, providers/mondoo)
+	@$(call buildProviderDist, providers/nmap)
 
 providers/bundle:
 	@$(call bundleProvider, providers/network)
@@ -442,6 +450,7 @@ providers/bundle:
 	@$(call bundleProvider, providers/ansible)
 	@$(call bundleProvider, providers/snowflake)
 	@$(call bundleProvider, providers/mondoo)
+	@$(call bundleProvider, providers/nmap)
 
 providers/test:
 	@$(call testProvider, providers/core)
@@ -471,6 +480,7 @@ providers/test:
 	@$(call testGoModProvider, providers/ansible)
 	@$(call testGoModProvider, providers/snowflake)
 	@$(call testGoModProvider, providers/mondoo)
+	@$(call testGoModProvider, providers/nmap)
 
 lr/test:
 	go test ./resources/lr/...
@@ -564,6 +574,11 @@ lr/docs/markdown: providers/lr
 		--description "The Network resource pack lets you use MQL to query and assess the security of domains and network services." \
 		--docs-file providers/network/resources/network.lr.manifest.yaml \
 		--output ../docs/docs/mql/resources/network-pack
+	./lr markdown providers/network/resources/nmap.lr \
+		--pack-name "nmap" \
+		--description "The Nmap resource pack lets you use MQL to query and assess Nmap data." \
+		--docs-file providers/network/resources/nmap.lr.manifest.yaml \
+		--output ../docs/docs/mql/resources/nmap-pack
 	./lr markdown providers/oci/resources/oci.lr \
 		--pack-name "Oracle Cloud Infrastructure (OCI)" \
 		--description "The Oracle Cloud Infrastructure (OCI) resource pack lets you use MQL to query and assess the security of your OCI services." \
@@ -624,6 +639,12 @@ lr/docs/markdown: providers/lr
 		--description "The Cloudflare resource pack lets you use MQL to query and assess the security of your Cloudflare configuration." \
 		--docs-file providers/cloudflare/resources/cloudflare.lr.manifest.yaml \
 		--output ../docs/docs/mql/resources/cloudflare-pack
+	./lr markdown providers/nmap/resources/nmap.lr \
+		--pack-name "Nmap" \
+		--description "The Nmap resource pack lets you use MQL to query and assess the network devices with Nmap." \
+		--docs-file providers/nmap/resources/nmap.lr.manifest.yaml \
+		--output ../docs/docs/mql/resources/nmap-pack
+
 
 lr/docs/stats:
 	@echo "Please remember to re-run before using this:"
@@ -700,6 +721,10 @@ test: test/go test/lint
 benchmark/go:
 	go test -bench=. -benchmem go.mondoo.com/cnquery/v11/explorer/scan/benchmark
 
+race/go:
+	go test -race go.mondoo.com/cnquery/v11/internal/workerpool
+	go test -race go.mondoo.com/cnquery/v11/explorer/scan
+
 test/generate: prep/tools/mockgen
 	go generate ./providers
 

explorer/scan/discovery.go

@@ -6,11 +6,13 @@ package scan
 import (
 	"context"
 	"errors"
+	"sync"
 	"time"
 
 	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/v11/cli/config"
 	"go.mondoo.com/cnquery/v11/cli/execruntime"
+	"go.mondoo.com/cnquery/v11/internal/workerpool"
 	"go.mondoo.com/cnquery/v11/llx"
 	"go.mondoo.com/cnquery/v11/logger"
 	"go.mondoo.com/cnquery/v11/providers"
@@ -20,6 +22,9 @@ import (
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/upstream"
 )
 
+// number of parallel goroutines discovering assets
+const workers = 10
+
 type AssetWithRuntime struct {
 	Asset   *inventory.Asset
 	Runtime *providers.Runtime
@@ -34,28 +39,30 @@ type DiscoveredAssets struct {
 	platformIds map[string]struct{}
 	Assets      []*AssetWithRuntime
 	Errors      []*AssetWithError
+	assetsLock  sync.Mutex
 }
 
 // Add adds an asset and its runtime to the discovered assets list. It returns true if the
 // asset has been added, false if it is a duplicate
 func (d *DiscoveredAssets) Add(asset *inventory.Asset, runtime *providers.Runtime) bool {
-	isDuplicate := false
+	d.assetsLock.Lock()
+	defer d.assetsLock.Unlock()
+
 	for _, platformId := range asset.PlatformIds {
 		if _, ok := d.platformIds[platformId]; ok {
-			isDuplicate = true
-			break
+			// duplicate
+			return false
 		}
 		d.platformIds[platformId] = struct{}{}
 	}
-	if isDuplicate {
-		return false
-	}
 
 	d.Assets = append(d.Assets, &AssetWithRuntime{Asset: asset, Runtime: runtime})
 	return true
 }
 
 func (d *DiscoveredAssets) AddError(asset *inventory.Asset, err error) {
+	d.assetsLock.Lock()
+	defer d.assetsLock.Unlock()
 	d.Errors = append(d.Errors, &AssetWithError{Asset: asset, Err: err})
 }
 
@@ -161,17 +168,30 @@ func discoverAssets(rootAssetWithRuntime *AssetWithRuntime, resolvedRootAsset *i
 		return
 	}
 
+	pool := workerpool.New[*AssetWithRuntime](workers)
+	pool.Start()
+	defer pool.Close()
+
 	// for all discovered assets, we apply mondoo-specific labels and annotations that come from the root asset
-	for _, a := range rootAssetWithRuntime.Runtime.Provider.Connection.Inventory.Spec.Assets {
-		// create runtime for root asset
-		assetWithRuntime, err := createRuntimeForAsset(a, upstream, recording)
-		if err != nil {
-			log.Error().Err(err).Str("asset", a.Name).Msg("unable to create runtime for asset")
-			discoveredAssets.AddError(a, err)
-			continue
-		}
+	for _, asset := range rootAssetWithRuntime.Runtime.Provider.Connection.Inventory.Spec.Assets {
+		pool.Submit(func() (*AssetWithRuntime, error) {
+			assetWithRuntime, err := createRuntimeForAsset(asset, upstream, recording)
+			if err != nil {
+				log.Error().Err(err).Str("asset", asset.GetName()).Msg("unable to create runtime for asset")
+				discoveredAssets.AddError(asset, err)
+			}
+			return assetWithRuntime, nil
+		})
+	}
+
+	// Wait for the workers to finish processing
+	pool.Wait()
+
+	// Get all assets with runtimes from the pool
+	for _, result := range pool.GetResults() {
+		assetWithRuntime := result.Value
 
-		// If no asset was returned and no error, then we observed a duplicate asset with a
+		// If asset is nil, then we observed a duplicate asset with a
 		// runtime that already exists.
 		if assetWithRuntime == nil {
 			continue

internal/workerpool/collector.go

@@ -9,13 +9,11 @@ import (
 )
 
 type collector[R any] struct {
-	resultsCh <-chan R
-	results   []R
+	resultsCh <-chan Result[R]
+	results   []Result[R]
 	read      sync.Mutex
 
-	errorsCh <-chan error
-	errors   []error
-
+	// The total number of requests read.
 	requestsRead int64
 }
 
@@ -27,29 +25,35 @@ func (c *collector[R]) start() {
 				c.read.Lock()
 				c.results = append(c.results, result)
 				c.read.Unlock()
-
-			case err := <-c.errorsCh:
-				c.read.Lock()
-				c.errors = append(c.errors, err)
-				c.read.Unlock()
 			}
 
 			atomic.AddInt64(&c.requestsRead, 1)
 		}
 	}()
 }
-func (c *collector[R]) GetResults() []R {
+
+func (c *collector[R]) RequestsRead() int64 {
+	return atomic.LoadInt64(&c.requestsRead)
+}
+
+func (c *collector[R]) GetResults() []Result[R] {
 	c.read.Lock()
 	defer c.read.Unlock()
 	return c.results
 }
 
-func (c *collector[R]) GetErrors() []error {
-	c.read.Lock()
-	defer c.read.Unlock()
-	return c.errors
+func (c *collector[R]) GetValues() (slice []R) {
+	results := c.GetResults()
+	for i := range results {
+		slice = append(slice, results[i].Value)
+	}
+	return
 }
 
-func (c *collector[R]) RequestsRead() int64 {
-	return atomic.LoadInt64(&c.requestsRead)
+func (c *collector[R]) GetErrors() (slice []error) {
+	results := c.GetResults()
+	for i := range results {
+		slice = append(slice, results[i].Error)
+	}
+	return
 }