Skip to content

Commit

Permalink
⭐️ SBOM connection (#3704)
Browse files Browse the repository at this point in the history
* 🧹 refactor sbom package

* ⭐️ parse different sbom formats

* 🧹 extract io writer to iox package

* 🧹 improve recordings

* 🐛 fix makefile order of generate

provider build always need to happen last

* ⭐️ support piped input

* ⭐️ sbom connection support

* 🧹 update go mod

* Update sbom/cyclonedx.go

Co-authored-by: Scott Ford <[email protected]>

---------

Co-authored-by: Scott Ford <[email protected]>
  • Loading branch information
chris-rock and scottford-io authored May 30, 2024
1 parent 7766f9c commit 23431c4
Show file tree
Hide file tree
Showing 57 changed files with 3,502 additions and 408 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ prep/tools/mockgen:

# 🌙 MQL/MOTOR #

cnquery/generate: clean/proto llx/generate shared/generate providers explorer/generate sbom/generate reporter/generate
cnquery/generate: clean/proto llx/generate shared/generate explorer/generate sbom/generate reporter/generate providers

cnquery/generate/core: clean/proto llx/generate shared/generate providers/proto providers/build/mock providers/build/core explorer/generate sbom/generate reporter/generate

Expand Down Expand Up @@ -648,7 +648,7 @@ test/lint/golangci-lint/run: prep/tools
golangci-lint --version
golangci-lint run

test/lint/extended: prep/tools
test/lint/extended: prep/tools test/generate
golangci-lint run --config=.github/.golangci.yml --timeout=30m

test/lint/proto: prep/tools/protolint
Expand Down
3 changes: 2 additions & 1 deletion apps/cnquery/cmd/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"go.mondoo.com/cnquery/v11/providers-sdk/v1/upstream"
"go.mondoo.com/cnquery/v11/shared"
run "go.mondoo.com/cnquery/v11/shared/proto"
"go.mondoo.com/cnquery/v11/utils/iox"
"google.golang.org/protobuf/proto"
)

Expand Down Expand Up @@ -52,7 +53,7 @@ func init() {

type cnqueryPlugin struct{}

func (c *cnqueryPlugin) RunQuery(conf *run.RunQueryConfig, runtime *providers.Runtime, out shared.OutputHelper) error {
func (c *cnqueryPlugin) RunQuery(conf *run.RunQueryConfig, runtime *providers.Runtime, out iox.OutputHelper) error {
if conf.Command == "" && conf.Input == "" {
return errors.New("No command provided, nothing to do.")
}
Expand Down
2 changes: 1 addition & 1 deletion apps/cnquery/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ func BuildRootCmd() (*cobra.Command, error) {
Command: sbomCmd,
Run: sbomCmdRun,
Action: "Collect a software bill of materials (SBOM) for ",
SupportedConnectors: []string{"docker", "container", "filesystem", "local", "ssh", "vagrant", "winrm"},
SupportedConnectors: []string{"docker", "container", "filesystem", "local", "ssh", "vagrant", "winrm", "sbom"},
},
)
return rootCmd, err
Expand Down
4 changes: 2 additions & 2 deletions apps/cnquery/cmd/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (
"go.mondoo.com/cnquery/v11/providers"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v11/shared"
"go.mondoo.com/cnquery/v11/shared/proto"
"go.mondoo.com/cnquery/v11/utils/iox"
)

func init() {
Expand Down Expand Up @@ -82,7 +82,7 @@ var RunCmdRun = func(cmd *cobra.Command, runtime *providers.Runtime, cliRes *plu
conf.Incognito, _ = cmd.Flags().GetBool("incognito")

x := cnqueryPlugin{}
w := shared.IOWriter{Writer: os.Stdout}
w := iox.IOWriter{Writer: os.Stdout}
err = x.RunQuery(&conf, runtime, &w)
if err != nil {
log.Fatal().Err(err).Msg("failed to run query")
Expand Down
10 changes: 6 additions & 4 deletions apps/cnquery/cmd/sbom.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"go.mondoo.com/cnquery/v11/providers"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
sbom "go.mondoo.com/cnquery/v11/sbom"
"go.mondoo.com/cnquery/v11/sbom/generator"
"go.mondoo.com/cnquery/v11/sbom/pack"
)

func init() {
Expand Down Expand Up @@ -66,7 +68,7 @@ Note this command is experimental and may change in the future.

var sbomCmdRun = func(cmd *cobra.Command, runtime *providers.Runtime, cliRes *plugin.ParseCLIRes) {
log.Info().Msg("This command is experimental. Please report any issues to https://github.com/mondoohq/cnquery.")
pb, err := sbom.QueryPack()
pb, err := pack.QueryPack()
if err != nil {
log.Fatal().Err(err).Msg("failed to load query pack")
}
Expand All @@ -93,14 +95,14 @@ var sbomCmdRun = func(cmd *cobra.Command, runtime *providers.Runtime, cliRes *pl
logger.DebugDumpJSON("mondoo-sbom-report", data)
}

boms, err := sbom.NewBom(cnspecReport)
boms, err := generator.NewBom(cnspecReport)
if err != nil {
log.Fatal().Err(err).Msg("failed to parse bom")
}

var exporter sbom.Exporter
var exporter sbom.FormatSpecificationHandler
output := viper.GetString("output")
exporter = sbom.NewExporter(output)
exporter = sbom.New(output)
if exporter == nil {
log.Fatal().Err(err).Msg("failed to get exporter for output format: " + output)
}
Expand Down
40 changes: 2 additions & 38 deletions cli/inventoryloader/inventory.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,7 @@ package inventoryloader

import (
"bytes"
"io"
"os"
"runtime"
"text/template"

"github.com/cockroachdb/errors"
Expand All @@ -16,43 +14,9 @@ import (
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory/ansibleinventory"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory/domainlist"
"go.mondoo.com/cnquery/v11/utils/piped"
)

func loadDataPipe() ([]byte, bool) {
isTerminal := true
isNamedPipe := false
switch runtime.GOOS {
case "darwin", "dragonfly", "netbsd", "solaris", "linux":
// when we run the following command, the detection differs between macos and linux
// cat options.json | mondoo scan
// for macos, we get isNamedPipe=false, isTerminal=false, size > 0
// but this only applies to direct terminal execution, for the same command in a bash file, we get
// for macos bash script, we get isNamedPipe=true, isTerminal=false, size > 0
// for linux, we get isNamedPipe=true, isTerminal=false, size=0
// Therefore we always want to check for file size if we detected its not a terminal
// If we are not checking for fi.Size() > 0 even a run inside of a bash script turn out
// to be pipes, therefore we need to verify that there is some data available at the pipe
// also read https://flaviocopes.com/go-shell-pipes/
fi, _ := os.Stdin.Stat()
isTerminal = (fi.Mode() & os.ModeCharDevice) == os.ModeCharDevice
isNamedPipe = (fi.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe
log.Debug().Bool("isTerminal", isTerminal).Bool("isNamedPipe", isNamedPipe).Int64("size", fi.Size()).Msg("check if we got the scan config from pipe")
if isNamedPipe || (!isTerminal && fi.Size() > 0) {
// Pipe input
log.Debug().Msg("read scan config from stdin pipe")

// read stdin into buffer
data, err := io.ReadAll(os.Stdin)
if err != nil {
log.Error().Err(err).Msg("could not read from pipe")
return nil, false
}
return data, true
}
}
return nil, false
}

func renderTemplate(data []byte) ([]byte, error) {
type InventoryTemplateVariables struct{}
conf := InventoryTemplateVariables{}
Expand Down Expand Up @@ -92,7 +56,7 @@ func Parse() (*inventory.Inventory, error) {
// read data from stdin
log.Info().Msg("load inventory from piped input")
var ok bool
data, ok = loadDataPipe()
data, ok = piped.LoadDataFromPipe()
if !ok {
return nil, errors.New("could not read inventory from piped input")
}
Expand Down
40 changes: 40 additions & 0 deletions cli/providers/providers.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package providers

import (
"encoding/json"
"go.mondoo.com/cnquery/v11/utils/piped"
"go.mondoo.com/ranger-rpc/status"
"os"
"strings"
Expand Down Expand Up @@ -491,9 +492,42 @@ func setConnector(provider *plugin.Provider, connector *plugin.Connector, run fu
return // adding this here as a compiler hint to stop warning about nil-dereferences
}

temporaryFiles := []string{}
if cliRes.Asset == nil {
log.Warn().Err(err).Msg("failed to discover assets after processing CLI arguments")
} else {
// if we have an asset, we check if the path requires piped content
for _, conf := range cliRes.Asset.Connections {
if conf.Path != "-" {
continue
}

// we need to read from stdin
data, ok := piped.LoadDataFromPipe()
if !ok {
log.Fatal().Msg("could not load data from piped input")
}
// generate a new temporary file and write the content to it
tmpFile, err := os.CreateTemp("", "cnquery-stdin-")
if err != nil {
log.Fatal().Err(err).Msg("could not create temporary file for stdin")
}
// remove temporary file after the runtime is closed
log.Info().Str("file", tmpFile.Name()).Msg("created temporary file for stdin")
temporaryFiles = append(temporaryFiles, tmpFile.Name())
if _, err := tmpFile.Write(data); err != nil {
log.Fatal().Err(err).Str("file", tmpFile.Name()).Msg("could not write data to temporary file")
}

// replace path with the temporary file and set a flag to indicate that it is piped content
conf.Path = tmpFile.Name()
if conf.Options == nil {
conf.Options = map[string]string{}
}
conf.Options["piped"] = "true"
}

// NOTE: no fatal output beyond this point, so that the runtime is closed properly
assetRuntime, err := providers.Coordinator.RuntimeFor(cliRes.Asset, runtime)
if err != nil {
log.Warn().Err(err).Msg("failed to get runtime for an asset that was detected after parsing the CLI")
Expand All @@ -505,6 +539,12 @@ func setConnector(provider *plugin.Provider, connector *plugin.Connector, run fu
run(cc, runtime, cliRes)
runtime.Close()
providers.Coordinator.Shutdown()

// remove optional temporary files
for _, tmpFile := range temporaryFiles {
_ = os.Remove(tmpFile)
}

}

attachFlags(cmd.Flags(), allFlags)
Expand Down
8 changes: 4 additions & 4 deletions cli/reporter/csv.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"go.mondoo.com/cnquery/v11/explorer"
"go.mondoo.com/cnquery/v11/llx"
"go.mondoo.com/cnquery/v11/mrn"
"go.mondoo.com/cnquery/v11/shared"
"go.mondoo.com/cnquery/v11/utils/iox"
)

type csvStruct struct {
Expand All @@ -28,7 +28,7 @@ func (c csvStruct) toSlice() []string {
}

// ConvertToCSV writes the given report collection to the given output directory
func ConvertToCSV(data *explorer.ReportCollection, out shared.OutputHelper) error {
func ConvertToCSV(data *explorer.ReportCollection, out iox.OutputHelper) error {
w := csv.NewWriter(out)

// write header
Expand Down Expand Up @@ -93,7 +93,7 @@ func ConvertToCSV(data *explorer.ReportCollection, out shared.OutputHelper) erro
if resolvedPack != nil && resolvedPack.ExecutionJob != nil {
for qid, query := range resolvedPack.ExecutionJob.Queries {
buf := &bytes.Buffer{}
resultWriter := &shared.IOWriter{Writer: buf}
resultWriter := &iox.IOWriter{Writer: buf}
err := ResultsToCsvEntry(query.Code, results, resultWriter)
if err != nil {
return err
Expand Down Expand Up @@ -131,7 +131,7 @@ func ConvertToCSV(data *explorer.ReportCollection, out shared.OutputHelper) erro
return w.Error()
}

func ResultsToCsvEntry(code *llx.CodeBundle, results map[string]*llx.RawResult, out shared.OutputHelper) error {
func ResultsToCsvEntry(code *llx.CodeBundle, results map[string]*llx.RawResult, out iox.OutputHelper) error {
var checksums []string
eps := code.CodeV2.Entrypoints()
checksums = make([]string, len(eps))
Expand Down
4 changes: 2 additions & 2 deletions cli/reporter/csv_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (

"github.com/stretchr/testify/require"
"go.mondoo.com/cnquery/v11/explorer"
"go.mondoo.com/cnquery/v11/shared"
"go.mondoo.com/cnquery/v11/utils/iox"
"sigs.k8s.io/yaml"
)

Expand All @@ -20,7 +20,7 @@ func TestCSVExport(t *testing.T) {
var report *explorer.ReportCollection
err = yaml.Unmarshal(data, &report)
require.NoError(t, err)
w := shared.IOWriter{Writer: os.Stdout}
w := iox.IOWriter{Writer: os.Stdout}
err = ConvertToCSV(report, &w)
require.NoError(t, err)
}
7 changes: 4 additions & 3 deletions cli/reporter/json.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@ package reporter
import (
"encoding/json"
"errors"

"go.mondoo.com/cnquery/v11/explorer"
"go.mondoo.com/cnquery/v11/llx"
"go.mondoo.com/cnquery/v11/shared"
"go.mondoo.com/cnquery/v11/utils/iox"
)

func CodeBundleToJSON(code *llx.CodeBundle, results map[string]*llx.RawResult, out shared.OutputHelper) error {
func CodeBundleToJSON(code *llx.CodeBundle, results map[string]*llx.RawResult, out iox.OutputHelper) error {
var checksums []string
eps := code.CodeV2.Entrypoints()
checksums = make([]string, len(eps))
Expand Down Expand Up @@ -43,7 +44,7 @@ func CodeBundleToJSON(code *llx.CodeBundle, results map[string]*llx.RawResult, o
return nil
}

func ConvertToJSON(data *explorer.ReportCollection, out shared.OutputHelper) error {
func ConvertToJSON(data *explorer.ReportCollection, out iox.OutputHelper) error {
if data == nil {
return nil
}
Expand Down
10 changes: 5 additions & 5 deletions cli/reporter/json_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@ package reporter

import (
"bytes"
"go.mondoo.com/cnquery/v11/explorer"
"os"
"sigs.k8s.io/yaml"
"strings"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.mondoo.com/cnquery/v11/explorer"
"go.mondoo.com/cnquery/v11/llx"
"go.mondoo.com/cnquery/v11/providers-sdk/v1/testutils"
"go.mondoo.com/cnquery/v11/shared"
"go.mondoo.com/cnquery/v11/utils/iox"
"sigs.k8s.io/yaml"
)

var x = testutils.InitTester(testutils.LinuxMock())
Expand Down Expand Up @@ -44,7 +44,7 @@ func TestQueryConversion(t *testing.T) {
}

var out strings.Builder
w := shared.IOWriter{Writer: &out}
w := iox.IOWriter{Writer: &out}

for i := range tests {
cur := tests[i]
Expand All @@ -66,7 +66,7 @@ func TestJsonReporter(t *testing.T) {
require.NoError(t, err)

buf := &bytes.Buffer{}
w := shared.IOWriter{Writer: buf}
w := iox.IOWriter{Writer: buf}
err = ConvertToJSON(report, &w)
require.NoError(t, err)
}
6 changes: 3 additions & 3 deletions cli/reporter/proto.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,11 @@ import (
"bytes"
"encoding/json"
"errors"
"google.golang.org/protobuf/encoding/protojson"
"strings"

"go.mondoo.com/cnquery/v11/explorer"
"go.mondoo.com/cnquery/v11/shared"
"go.mondoo.com/cnquery/v11/utils/iox"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"
)

Expand Down Expand Up @@ -68,7 +68,7 @@ func ConvertToProto(data *explorer.ReportCollection) (*Report, error) {
queryID := prettyPrintString(printID)

buf := &bytes.Buffer{}
w := shared.IOWriter{Writer: buf}
w := iox.IOWriter{Writer: buf}
err := CodeBundleToJSON(query.Code, results, &w)
if err != nil {
return nil, err
Expand Down
Loading

0 comments on commit 23431c4

Please sign in to comment.