⭐️ integration tests for cnquery + providers + mql queries (#3359)

.github/workflows/pr-test-lint.yml

@@ -95,7 +95,7 @@ jobs:
           name: test-results
           path: '*.xml'
 
-  go-test-cli:
+  go-test-integration:
     runs-on: self-hosted
     timeout-minutes: 120
     steps:
@@ -119,8 +119,8 @@ jobs:
       - name: Display Provider PAth
         run: echo $PROVIDERS_PATH
 
-      - name: Test cnquery CLI
-        run: make test/go-cli/plain-ci
+      - name: Test cnquery CLI and Providers
+        run: make test/integration
 
       - uses: actions/upload-artifact@v4  # upload test results
         if: success() || failure()        # run this step even if previous step failed

Makefile

@@ -605,16 +605,16 @@ test/generate: prep/tools/mockgen
 test/go: cnquery/generate test/generate test/go/plain
 
 test/go/plain:
-	go test -cover $(shell go list ./... | grep -v '/providers/' | grep -v '/test/cli')
+	go test -cover $(shell go list ./... | grep -v '/providers/' | grep -v '/test/')
 
 test/go/plain-ci: prep/tools test/generate providers/build
-	gotestsum --junitfile report.xml --format pkgname -- -cover $(shell go list ./... | grep -v '/vendor/' | grep -v '/providers/' | grep -v '/test/cli')
+	gotestsum --junitfile report.xml --format pkgname -- -cover $(shell go list ./... | grep -v '/vendor/' | grep -v '/providers/' | grep -v '/test/')
 
-test/go-cli/plain:
-	go test -cover $(shell go list ./... | grep 'test/cli')
+test/integration:
+	go test -cover $(shell go list ./... | grep '/test/')
 
 test/go-cli/plain-ci: prep/tools test/generate providers/build
-	gotestsum --junitfile report.xml --format pkgname -- -cover $(shell go list ./... | grep 'test/cli')
+	gotestsum --junitfile report.xml --format pkgname -- -cover $(shell go list ./... | grep '/test/')
 
 .PHONY: test/lint/staticcheck
 test/lint/staticcheck:

test/cli_test_runner.go

@@ -0,0 +1,72 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package test
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+)
+
+type Runner interface {
+	Run() error
+	ExitCode() int
+	Stdout() []byte
+	Stderr() []byte
+	Json(v any) error
+}
+
+type cliTestRunner struct {
+	cmd    *exec.Cmd
+	binary string
+	args   []string
+	stdout bytes.Buffer
+	stderr bytes.Buffer
+}
+
+func NewCliTestRunner(binary string, args ...string) *cliTestRunner {
+	c := &cliTestRunner{
+		binary: binary,
+		args:   args,
+	}
+	return c
+}
+
+func (c *cliTestRunner) Run() error {
+	c.cmd = exec.Command(c.binary, c.args...)
+	c.cmd.Stdout = &c.stdout
+	c.cmd.Stderr = &c.stderr
+
+	if err := c.cmd.Start(); err != nil {
+		return fmt.Errorf("Error starting command: %s\n", err)
+	}
+
+	// Wait for the command to finish
+	if err := c.cmd.Wait(); err != nil {
+		if _, ok := err.(*exec.ExitError); ok {
+			// The program has exited with an exit code != 0, but for testing purposes we don't want to fail the test
+			return nil
+		}
+		return fmt.Errorf("Command finished with error: %v\n", err)
+	}
+
+	return nil
+}
+
+func (c *cliTestRunner) ExitCode() int {
+	return c.cmd.ProcessState.ExitCode()
+}
+
+func (c *cliTestRunner) Stdout() []byte {
+	return c.stdout.Bytes()
+}
+
+func (c *cliTestRunner) Stderr() []byte {
+	return c.stderr.Bytes()
+}
+
+func (c *cliTestRunner) Json(v any) error {
+	return json.Unmarshal(c.Stdout(), v)
+}

test/cli/cli_test.go → test/commands/cli_test.go

@@ -1,7 +1,7 @@
 // Copyright (c) Mondoo, Inc.
 // SPDX-License-Identifier: BUSL-1.1
 
-package cli
+package commands
 
 import (
 	"os"

test/providers/.gitignore

@@ -0,0 +1 @@
+cnquery

test/providers/os_test.go

@@ -0,0 +1,187 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package providers
+
+import (
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mondoo.com/cnquery/v10/test"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"testing"
+)
+
+var once sync.Once
+
+// setup builds cnquery locally
+func setup() {
+	// build cnspec
+	if err := exec.Command("go", "build", "../../apps/cnquery/cnquery.go").Run(); err != nil {
+		log.Fatalf("building cnquery: %v", err)
+	}
+
+	// install local provider
+	if err := exec.Command("bash", "-c", "cd ../.. && make providers/build/os providers/install/os").Run(); err != nil {
+		log.Fatalf("building os provider: %v", err)
+	}
+
+	providersPATH := os.Getenv("PROVIDERS_PATH")
+	if providersPATH != "" {
+
+		// provider install places the provider in the "$(HOME)/.config/mondoo/providers/${$@_NAME}") but we
+		// want to test it in isolation. Therefore, we copy the provider to the current directory .providers
+		osProviderPath := filepath.Join(providersPATH, "os")
+		if err := os.MkdirAll(osProviderPath, 0755); err != nil {
+			log.Fatalf("creating directory: %v", err)
+		}
+
+		if err := exec.Command("cp", "-r", os.ExpandEnv("../../providers/os/dist"), osProviderPath).Run(); err != nil {
+			log.Fatalf("copying provider: %v", err)
+		}
+	}
+}
+
+const mqlPackagesQuery = "packages"
+
+type mqlPackages []struct {
+	Packages []struct {
+		Name    string `json:"name,omitempty"`
+		Version string `json:"version,omitempty"`
+	} `json:"packages.list,omitempty"`
+}
+
+const mqlPlatformQuery = "asset.platform"
+
+type mqlPlatform []struct {
+	Platform string `json:"asset.platform,omitempty"`
+}
+
+type connections []struct {
+	name   string
+	binary string
+	args   []string
+	tests  []mqlTest
+}
+
+type mqlTest struct {
+	query    string
+	expected func(*testing.T, test.Runner)
+}
+
+func TestOsProviderSharedTests(t *testing.T) {
+	once.Do(setup)
+
+	connections := connections{
+		{
+			name:   "local",
+			binary: "./cnquery",
+			args:   []string{"run", "local"},
+			tests: []mqlTest{
+				{
+					mqlPackagesQuery,
+					func(t *testing.T, r test.Runner) {
+						var c mqlPackages
+						err := r.Json(&c)
+						assert.NoError(t, err)
+
+						x := c[0]
+						assert.NotNil(t, x.Packages)
+						assert.True(t, len(x.Packages) > 0)
+					},
+				},
+				{
+					mqlPlatformQuery,
+					func(t *testing.T, r test.Runner) {
+						var c mqlPlatform
+						err := r.Json(&c)
+						assert.NoError(t, err)
+
+						x := c[0]
+						assert.True(t, len(x.Platform) > 0)
+					},
+				},
+			},
+		},
+		{
+			name:   "fs",
+			binary: "./cnquery",
+			args:   []string{"run", "fs", "--path", "./testdata/fs"},
+			tests: []mqlTest{
+				{
+					mqlPackagesQuery,
+					func(t *testing.T, r test.Runner) {
+						var c mqlPackages
+						err := r.Json(&c)
+						assert.NoError(t, err)
+
+						x := c[0]
+						assert.NotNil(t, x.Packages)
+						assert.True(t, len(x.Packages) > 0)
+					},
+				},
+				{
+					mqlPlatformQuery,
+					func(t *testing.T, r test.Runner) {
+						var c mqlPlatform
+						err := r.Json(&c)
+						assert.NoError(t, err)
+
+						x := c[0]
+						assert.Equal(t, "debian", x.Platform)
+					},
+				},
+			},
+		},
+		{
+			name:   "docker",
+			binary: "./cnquery",
+			args:   []string{"run", "docker", "alpine:latest"},
+			tests: []mqlTest{
+				{
+					mqlPackagesQuery,
+					func(t *testing.T, r test.Runner) {
+						var c mqlPackages
+						err := r.Json(&c)
+						assert.NoError(t, err)
+
+						x := c[0]
+						assert.NotNil(t, x.Packages)
+						assert.True(t, len(x.Packages) > 0)
+					},
+				},
+				{
+					mqlPlatformQuery,
+					func(t *testing.T, r test.Runner) {
+						var c mqlPlatform
+						err := r.Json(&c)
+						assert.NoError(t, err)
+
+						x := c[0]
+						assert.Equal(t, "alpine", x.Platform)
+					},
+				},
+			},
+		},
+	}
+
+	// iterate over all tests for all connections
+	for _, cc := range connections {
+		for _, tt := range cc.tests {
+
+			t.Run(cc.name+"/"+tt.query, func(t *testing.T) {
+				r := test.NewCliTestRunner(cc.binary, append(cc.args, "-c", tt.query, "-j")...)
+				err := r.Run()
+				require.NoError(t, err)
+				assert.Equal(t, 0, r.ExitCode())
+				assert.NotNil(t, r.Stdout())
+				assert.NotNil(t, r.Stderr())
+
+				tt.expected(t, r)
+			})
+		}
+	}
+}

test/providers/testdata/fs/etc/debian_version

@@ -0,0 +1 @@
+12.5

test/providers/testdata/fs/etc/hostname

@@ -0,0 +1 @@
+debianfs

test/providers/testdata/fs/etc/os-release

@@ -0,0 +1,9 @@
+PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
+NAME="Debian GNU/Linux"
+VERSION_ID="12"
+VERSION="12 (bookworm)"
+VERSION_CODENAME=bookworm
+ID=debian
+HOME_URL="https://www.debian.org/"
+SUPPORT_URL="https://www.debian.org/support"
+BUG_REPORT_URL="https://bugs.debian.org/"

test/providers/testdata/fs/var/lib/dpkg/status

@@ -0,0 +1,49 @@
+Package: fdisk
+Status: install ok installed
+Priority: important
+Section: utils
+Installed-Size: 427
+Maintainer: Ubuntu Developers <[email protected]>
+Architecture: amd64
+Multi-Arch: foreign
+Source: util-linux
+Version: 2.31.1-0.4ubuntu3.1
+Replaces: util-linux (<< 2.30.1-0ubuntu4~)
+Depends: libc6 (>= 2.14), libfdisk1 (>= 2.31.1), libmount1 (>= 2.24.2), libncursesw5 (>= 6), libsmartcols1 (>= 2.28~rc1), libtinfo5 (>= 6)
+Breaks: util-linux (<< 2.30.1-0ubuntu4~)
+Description: collection of partitioning utilities
+ This package contains the classic fdisk, sfdisk and cfdisk partitioning
+ utilities from the util-linux suite.
+ .
+ The utilities included in this package allow you to partition
+ your hard disk. The utilities supports both modern and legacy
+ partition tables (eg. GPT, MBR, etc).
+ .
+ The fdisk utility is the classical text-mode utility.
+ The cfdisk utilitity gives a more userfriendly curses based interface.
+ The sfdisk utility is mostly for automation and scripting uses.
+Important: yes
+Original-Maintainer: LaMont Jones <[email protected]>
+
+Package: libpam-runtime
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 300
+Maintainer: Ubuntu Developers <[email protected]>
+Architecture: all
+Multi-Arch: foreign
+Source: pam
+Version: 1.1.8-3.6ubuntu2
+Replaces: libpam0g-dev, libpam0g-util
+Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
+Conflicts: libpam0g-util
+Conffiles:
+ /etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
+ /etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
+Description: Runtime support for the PAM library
+ Contains configuration files and  directories required for
+ authentication  to work on Debian systems.  This package is required
+ on almost all installations.
+Homepage: http://www.linux-pam.org/
+Original-Maintainer: Steve Langasek <[email protected]>