From 20e52642ff0186c34d490c1e53ed615be69be5d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:37:29 +0100 Subject: [PATCH 1/7] Bump alpine from 3.20 to 3.21 (#4962) Bumps alpine from 3.20 to 3.21. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1b624649f1..61634d7bf3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # Copyright (c) Mondoo, Inc. # SPDX-License-Identifier: BUSL-1.1 -FROM alpine:3.20 AS root +FROM alpine:3.21 AS root RUN apk update &&\ apk add ca-certificates wget tar &&\ rm -rf /var/cache/apk/* From ac14153566e8a233b900a6c954648edda5f07257 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 10 Dec 2024 13:09:07 +0100 Subject: [PATCH 2/7] =?UTF-8?q?=E2=9C=A8=20Update=20providers=2020241210?= =?UTF-8?q?=20(#4967)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mondoo Tools --- providers/ansible/config/config.go | 2 +- providers/arista/config/config.go | 2 +- providers/atlassian/config/config.go | 2 +- providers/aws/config/config.go | 2 +- providers/azure/config/config.go | 2 +- providers/cloudflare/config/config.go | 2 +- providers/cloudformation/config/config.go | 2 +- providers/core/config/config.go | 2 +- providers/equinix/config/config.go | 2 +- providers/gcp/config/config.go | 2 +- providers/github/config/config.go | 2 +- providers/gitlab/config/config.go | 2 +- providers/google-workspace/config/config.go | 2 +- providers/ipmi/config/config.go | 2 +- providers/k8s/config/config.go | 2 +- providers/mondoo/config/config.go | 2 +- providers/ms365/config/config.go | 2 +- providers/network/config/config.go | 2 +- providers/oci/config/config.go | 2 +- providers/okta/config/config.go | 2 +- providers/opcua/config/config.go | 2 +- providers/shodan/config/config.go | 2 +- providers/slack/config/config.go | 2 +- providers/snowflake/config/config.go | 2 +- providers/terraform/config/config.go | 2 +- providers/vcd/config/config.go | 2 +- providers/vsphere/config/config.go | 2 +- 27 files changed, 27 insertions(+), 27 deletions(-) diff --git a/providers/ansible/config/config.go b/providers/ansible/config/config.go index 0b3a3acb7c..ac9b4886d1 100644 --- a/providers/ansible/config/config.go +++ b/providers/ansible/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "ansible", ID: "go.mondoo.com/cnquery/v11/providers/ansible", - Version: "11.0.31", + Version: "11.0.32", ConnectionTypes: []string{provider.DefaultConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/arista/config/config.go b/providers/arista/config/config.go index 7972f7b837..ba4fdabc5e 100644 --- a/providers/arista/config/config.go +++ b/providers/arista/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "arista", ID: "go.mondoo.com/cnquery/v9/providers/arista", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/atlassian/config/config.go b/providers/atlassian/config/config.go index dbfc63e3df..75fa2c97f9 100644 --- a/providers/atlassian/config/config.go +++ b/providers/atlassian/config/config.go @@ -14,7 +14,7 @@ import ( var Config = plugin.Provider{ Name: "atlassian", ID: "go.mondoo.com/cnquery/v9/providers/atlassian", - Version: "11.0.47", + Version: "11.0.48", ConnectionTypes: []string{ provider.DefaultConnectionType, "jira", diff --git a/providers/aws/config/config.go b/providers/aws/config/config.go index b1b4a6bfbd..220559c35d 100644 --- a/providers/aws/config/config.go +++ b/providers/aws/config/config.go @@ -14,7 +14,7 @@ import ( var Config = plugin.Provider{ Name: "aws", ID: "go.mondoo.com/cnquery/v9/providers/aws", - Version: "11.5.10", + Version: "11.5.11", ConnectionTypes: []string{provider.DefaultConnectionType, string(awsec2ebsconn.EBSConnectionType)}, Connectors: []plugin.Connector{ { diff --git a/providers/azure/config/config.go b/providers/azure/config/config.go index ba116d6886..2a54174ae1 100644 --- a/providers/azure/config/config.go +++ b/providers/azure/config/config.go @@ -14,7 +14,7 @@ import ( var Config = plugin.Provider{ Name: "azure", ID: "go.mondoo.com/cnquery/v9/providers/azure", - Version: "11.3.18", + Version: "11.3.19", ConnectionTypes: []string{ provider.ConnectionType, string(azureinstancesnapshot.SnapshotConnectionType), diff --git a/providers/cloudflare/config/config.go b/providers/cloudflare/config/config.go index 9857453008..98ba26a562 100644 --- a/providers/cloudflare/config/config.go +++ b/providers/cloudflare/config/config.go @@ -11,7 +11,7 @@ import ( var Config = plugin.Provider{ Name: "cloudflare", ID: "go.mondoo.com/cnquery/v11/providers/cloudflare", - Version: "11.0.1", + Version: "11.0.2", ConnectionTypes: []string{provider.DefaultConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/cloudformation/config/config.go b/providers/cloudformation/config/config.go index 9deb880e0b..035e411d52 100644 --- a/providers/cloudformation/config/config.go +++ b/providers/cloudformation/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "cloudformation", ID: "go.mondoo.com/cnquery/v11/providers/cloudformation", - Version: "11.0.36", + Version: "11.0.37", ConnectionTypes: []string{provider.DefaultConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/core/config/config.go b/providers/core/config/config.go index 9943c0a511..7a5add0046 100644 --- a/providers/core/config/config.go +++ b/providers/core/config/config.go @@ -11,7 +11,7 @@ import ( var Config = plugin.Provider{ Name: "core", ID: "go.mondoo.com/cnquery/v9/providers/core", - Version: "11.0.23", + Version: "11.0.24", Connectors: []plugin.Connector{}, AssetUrlTrees: []*inventory.AssetUrlBranch{ { diff --git a/providers/equinix/config/config.go b/providers/equinix/config/config.go index e8de66852c..b20c73830c 100644 --- a/providers/equinix/config/config.go +++ b/providers/equinix/config/config.go @@ -11,7 +11,7 @@ import ( var Config = plugin.Provider{ Name: "equinix", ID: "go.mondoo.com/cnquery/v9/providers/equinix", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/gcp/config/config.go b/providers/gcp/config/config.go index abef2c6c92..54e64ec571 100644 --- a/providers/gcp/config/config.go +++ b/providers/gcp/config/config.go @@ -14,7 +14,7 @@ import ( var Config = plugin.Provider{ Name: "gcp", ID: "go.mondoo.com/cnquery/v9/providers/gcp", - Version: "11.0.50", + Version: "11.0.51", ConnectionTypes: []string{ provider.ConnectionType, string(gcpinstancesnapshot.SnapshotConnectionType), diff --git a/providers/github/config/config.go b/providers/github/config/config.go index 3d3b78bf4d..4cebccded3 100644 --- a/providers/github/config/config.go +++ b/providers/github/config/config.go @@ -13,7 +13,7 @@ import ( var Config = plugin.Provider{ Name: "github", ID: "go.mondoo.com/cnquery/v9/providers/github", - Version: "11.4.28", + Version: "11.4.29", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/gitlab/config/config.go b/providers/gitlab/config/config.go index 15d04cf5ca..3b6a6c1e8b 100644 --- a/providers/gitlab/config/config.go +++ b/providers/gitlab/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "gitlab", ID: "go.mondoo.com/cnquery/v9/providers/gitlab", - Version: "11.1.38", + Version: "11.1.39", ConnectionTypes: []string{ provider.ConnectionType, provider.GitlabGroupConnection, diff --git a/providers/google-workspace/config/config.go b/providers/google-workspace/config/config.go index f0e249142e..bbc3d5deba 100644 --- a/providers/google-workspace/config/config.go +++ b/providers/google-workspace/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "google-workspace", ID: "go.mondoo.com/cnquery/v9/providers/google-workspace", - Version: "11.1.22", + Version: "11.1.23", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/ipmi/config/config.go b/providers/ipmi/config/config.go index 8bded013c2..c27d209896 100644 --- a/providers/ipmi/config/config.go +++ b/providers/ipmi/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "ipmi", ID: "go.mondoo.com/cnquery/v9/providers/ipmi", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/k8s/config/config.go b/providers/k8s/config/config.go index dd785211a9..ab5fcf499c 100644 --- a/providers/k8s/config/config.go +++ b/providers/k8s/config/config.go @@ -13,7 +13,7 @@ import ( var Config = plugin.Provider{ Name: "k8s", ID: "go.mondoo.com/cnquery/v9/providers/k8s", - Version: "11.1.38", + Version: "11.1.39", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/mondoo/config/config.go b/providers/mondoo/config/config.go index 38e53d896e..88ab5f56de 100644 --- a/providers/mondoo/config/config.go +++ b/providers/mondoo/config/config.go @@ -11,7 +11,7 @@ import ( var Config = plugin.Provider{ Name: "mondoo", ID: "go.mondoo.com/cnquery/v11/providers/mondoo", - Version: "11.1.22", + Version: "11.1.23", ConnectionTypes: []string{provider.DefaultConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/ms365/config/config.go b/providers/ms365/config/config.go index 7edbd1ace8..907a6b5305 100644 --- a/providers/ms365/config/config.go +++ b/providers/ms365/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "ms365", ID: "go.mondoo.com/cnquery/v9/providers/ms365", - Version: "11.1.0", + Version: "11.1.1", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/network/config/config.go b/providers/network/config/config.go index 330d9446ed..da1d3157b8 100644 --- a/providers/network/config/config.go +++ b/providers/network/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "network", ID: "go.mondoo.com/cnquery/v9/providers/network", - Version: "11.0.39", + Version: "11.0.40", ConnectionTypes: []string{provider.HostConnectionType}, CrossProviderTypes: []string{ "go.mondoo.com/cnquery/providers/os", diff --git a/providers/oci/config/config.go b/providers/oci/config/config.go index df2c09b1ea..bd79c50622 100644 --- a/providers/oci/config/config.go +++ b/providers/oci/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "oci", ID: "go.mondoo.com/cnquery/v9/providers/oci", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/okta/config/config.go b/providers/okta/config/config.go index ee0e2377de..45fd65c497 100644 --- a/providers/okta/config/config.go +++ b/providers/okta/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "okta", ID: "go.mondoo.com/cnquery/v9/providers/okta", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/opcua/config/config.go b/providers/opcua/config/config.go index 62c08d2a3c..2830a78bf2 100644 --- a/providers/opcua/config/config.go +++ b/providers/opcua/config/config.go @@ -11,7 +11,7 @@ import ( var Config = plugin.Provider{ Name: "opcua", ID: "go.mondoo.com/cnquery/v9/providers/opcua", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/shodan/config/config.go b/providers/shodan/config/config.go index 73a1cafed5..55b5fa3beb 100644 --- a/providers/shodan/config/config.go +++ b/providers/shodan/config/config.go @@ -13,7 +13,7 @@ import ( var Config = plugin.Provider{ Name: "shodan", ID: "go.mondoo.com/cnquery/v11/providers/shodan", - Version: "11.0.33", + Version: "11.0.34", ConnectionTypes: []string{provider.DefaultConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/slack/config/config.go b/providers/slack/config/config.go index de1468efd6..99fa515280 100644 --- a/providers/slack/config/config.go +++ b/providers/slack/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "slack", ID: "go.mondoo.com/cnquery/v9/providers/slack", - Version: "11.0.46", + Version: "11.0.47", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/snowflake/config/config.go b/providers/snowflake/config/config.go index d185a0e875..73e588b00a 100644 --- a/providers/snowflake/config/config.go +++ b/providers/snowflake/config/config.go @@ -11,7 +11,7 @@ import ( var Config = plugin.Provider{ Name: "snowflake", ID: "go.mondoo.com/cnquery/v11/providers/snowflake", - Version: "11.0.30", + Version: "11.0.31", ConnectionTypes: []string{provider.DefaultConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/terraform/config/config.go b/providers/terraform/config/config.go index 4479de09d0..59f16d352c 100644 --- a/providers/terraform/config/config.go +++ b/providers/terraform/config/config.go @@ -12,7 +12,7 @@ import ( var Config = plugin.Provider{ Name: "terraform", ID: "go.mondoo.com/cnquery/v9/providers/terraform", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{ provider.StateConnectionType, provider.PlanConnectionType, diff --git a/providers/vcd/config/config.go b/providers/vcd/config/config.go index 96ce62787c..dc8f76123b 100644 --- a/providers/vcd/config/config.go +++ b/providers/vcd/config/config.go @@ -11,7 +11,7 @@ import ( var Config = plugin.Provider{ Name: "vcd", ID: "go.mondoo.com/cnquery/v9/providers/vcd", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { diff --git a/providers/vsphere/config/config.go b/providers/vsphere/config/config.go index a9c36de102..8aa20db739 100644 --- a/providers/vsphere/config/config.go +++ b/providers/vsphere/config/config.go @@ -13,7 +13,7 @@ import ( var Config = plugin.Provider{ Name: "vsphere", ID: "go.mondoo.com/cnquery/v9/providers/vsphere", - Version: "11.0.44", + Version: "11.0.45", ConnectionTypes: []string{provider.ConnectionType}, Connectors: []plugin.Connector{ { From c89b24c5489787735e2bc63d075a6b0c5e322b21 Mon Sep 17 00:00:00 2001 From: Christian Zunker <827818+czunker@users.noreply.github.com> Date: Tue, 10 Dec 2024 13:12:15 +0100 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=90=9B=20Checkout=20repo=20before=20a?= =?UTF-8?q?pproval=20(#4968)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian Zunker --- .github/workflows/pr-test-lint.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr-test-lint.yml b/.github/workflows/pr-test-lint.yml index b5bfc7f413..a46d51920b 100644 --- a/.github/workflows/pr-test-lint.yml +++ b/.github/workflows/pr-test-lint.yml @@ -184,6 +184,8 @@ jobs: contents: write pull-requests: write steps: + - name: Checkout code + uses: actions/checkout@v4 # figure out the PR for this commit - uses: cloudposse-github-actions/get-pr@v2.0.0 id: pr From c17e04867deef1922f12d27741ce0d9c757342cd Mon Sep 17 00:00:00 2001 From: Christian Zunker <827818+czunker@users.noreply.github.com> Date: Tue, 10 Dec 2024 16:18:53 +0100 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=A7=B9=20Bump=20go=20to=20v1.23.4=20(?= =?UTF-8?q?#4969)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian Zunker --- .github/env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/env b/.github/env index e6a968b9d6..dbffcb05c2 100644 --- a/.github/env +++ b/.github/env @@ -1 +1 @@ -golang-version=1.23.0 +golang-version=1.23.4 From ccf731df89796d032ce5ad620dbf5be0024f7419 Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Thu, 12 Dec 2024 16:17:30 +0200 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=90=9B=20fix=20find=20files=20for=20t?= =?UTF-8?q?ar=20(#4974)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 fix find files for tar Signed-off-by: Ivan Milchev * fix find by name Signed-off-by: Ivan Milchev --------- Signed-off-by: Ivan Milchev --- .../container/image_connection_test.go | 4 ++-- .../os/connection/tar/connection_test.go | 2 +- providers/os/connection/tar/fs.go | 22 +++++++++++++++++-- providers/os/fs/fs.go | 3 +++ providers/os/resources/files.go | 5 +++++ 5 files changed, 31 insertions(+), 5 deletions(-) diff --git a/providers/os/connection/container/image_connection_test.go b/providers/os/connection/container/image_connection_test.go index c9e17c612b..26c093f152 100644 --- a/providers/os/connection/container/image_connection_test.go +++ b/providers/os/connection/container/image_connection_test.go @@ -231,11 +231,11 @@ func TestImageConnections(t *testing.T) { fSearch := fs.(*tar.FS) if test.testfile == "/etc/alpine-release" { - infos, err := fSearch.Find("/", regexp.MustCompile(`alpine-release`), "file") + infos, err := fSearch.Find("/", regexp.MustCompile(`alpine-release`), "file", nil, nil) require.NoError(t, err) assert.Equal(t, 1, len(infos)) } else if test.testfile == "/etc/centos-release" { - infos, err := fSearch.Find("/", regexp.MustCompile(`centos-release`), "file") + infos, err := fSearch.Find("/", regexp.MustCompile(`centos-release`), "file", nil, nil) require.NoError(t, err) assert.Equal(t, 6, len(infos)) } diff --git a/providers/os/connection/tar/connection_test.go b/providers/os/connection/tar/connection_test.go index 8b85d2aceb..b192a2733f 100644 --- a/providers/os/connection/tar/connection_test.go +++ b/providers/os/connection/tar/connection_test.go @@ -236,7 +236,7 @@ func TestTarFileFind(t *testing.T) { fSearch := fs.(*tar.FS) - infos, err := fSearch.Find("/", regexp.MustCompile(`alpine-release`), "file") + infos, err := fSearch.Find("/", regexp.MustCompile(`alpine-release`), "file", nil, nil) require.NoError(t, err) assert.Equal(t, 1, len(infos)) diff --git a/providers/os/connection/tar/fs.go b/providers/os/connection/tar/fs.go index 323f3ae920..14de7b058c 100644 --- a/providers/os/connection/tar/fs.go +++ b/providers/os/connection/tar/fs.go @@ -16,9 +16,12 @@ import ( "github.com/rs/zerolog/log" "github.com/spf13/afero" + "go.mondoo.com/cnquery/v11/providers/os/connection/shared" "go.mondoo.com/cnquery/v11/providers/os/fsutil" ) +var _ shared.FileSearch = (*FS)(nil) + func NewFs(source string) *FS { return &FS{ Source: source, @@ -192,7 +195,7 @@ func (fs *FS) tar(path string, header *tar.Header) (io.ReadCloser, error) { // searches for files and returns the file info // regex can be nil -func (fs *FS) Find(from string, r *regexp.Regexp, typ string) ([]string, error) { +func (fs *FS) Find(from string, r *regexp.Regexp, typ string, perm *uint32, depth *int) ([]string, error) { list := []string{} for k := range fs.FileMap { p := strings.HasPrefix(k, from) @@ -200,10 +203,13 @@ func (fs *FS) Find(from string, r *regexp.Regexp, typ string) ([]string, error) if r != nil { m = r.MatchString(k) } + if !depthMatch(from, k, depth) { + continue + } log.Trace().Str("path", k).Str("from", from).Str("prefix", from).Bool("prefix", p).Bool("m", m).Msg("check if matches") if p && m { entry := fs.FileMap[k] - if (typ == "directory" && entry.Typeflag == tar.TypeDir) || (typ == "file" && entry.Typeflag == tar.TypeReg) { + if (typ == "directory" && entry.Typeflag == tar.TypeDir) || (typ == "file" && entry.Typeflag == tar.TypeReg) || typ == "" { list = append(list, k) log.Debug().Msg("matches") continue @@ -212,3 +218,15 @@ func (fs *FS) Find(from string, r *regexp.Regexp, typ string) ([]string, error) } return list, nil } + +func depthMatch(from, filepath string, depth *int) bool { + if depth == nil { + return true + } + + trimmed := strings.TrimPrefix(filepath, from) + // WalkDir always uses slash for separating, ignoring the OS separator. This is why we need to replace it. + normalized := strings.ReplaceAll(trimmed, string(os.PathSeparator), "/") + fileDepth := strings.Count(normalized, "/") + return fileDepth <= *depth +} diff --git a/providers/os/fs/fs.go b/providers/os/fs/fs.go index e9dd5891f2..41cd45a566 100644 --- a/providers/os/fs/fs.go +++ b/providers/os/fs/fs.go @@ -11,8 +11,11 @@ import ( "time" "github.com/spf13/afero" + "go.mondoo.com/cnquery/v11/providers/os/connection/shared" ) +var _ shared.FileSearch = (*MountedFs)(nil) + var notSupported = errors.New("not supported") type MountedFs struct { diff --git a/providers/os/resources/files.go b/providers/os/resources/files.go index 27ac507f21..6a5431e5a9 100644 --- a/providers/os/resources/files.go +++ b/providers/os/resources/files.go @@ -70,6 +70,11 @@ func (l *mqlFilesFind) list() ([]interface{}, error) { if err != nil { return nil, err } + } else if len(l.Name.Data) > 0 { + compiledRegexp, err = regexp.Compile(l.Name.Data) + if err != nil { + return nil, err + } } var foundFiles []string From ee1123c76d527a22ef5d4a154ce0f4d82dbff14b Mon Sep 17 00:00:00 2001 From: Ivan Milchev Date: Thu, 12 Dec 2024 16:23:04 +0200 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=8E=89=20os-11.3.13=20(#4975)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This release was created by cnquery's provider versioning bot. You can find me under: `providers-sdk/v1/util/version`. Co-authored-by: Mondoo --- providers/os/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/os/config/config.go b/providers/os/config/config.go index 13dafacb13..33a3d33916 100644 --- a/providers/os/config/config.go +++ b/providers/os/config/config.go @@ -13,7 +13,7 @@ import ( var Config = plugin.Provider{ Name: "os", ID: "go.mondoo.com/cnquery/v9/providers/os", - Version: "11.3.12", + Version: "11.3.13", ConnectionTypes: []string{ shared.Type_Local.String(), shared.Type_SSH.String(), From 949f0ceec75111bfb82736c14917a223cbdde9b1 Mon Sep 17 00:00:00 2001 From: Salim Afiune Maya Date: Thu, 12 Dec 2024 09:28:45 -0800 Subject: [PATCH 7/7] =?UTF-8?q?=E2=9A=A1=20fetch=20github=20org=20reposito?= =?UTF-8?q?ries=20in=20parallel=20=20(#4970)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introducing an internal go package called `workerpool` that we can use to send parallel requests when needed. :zap: For this change, I am making the fetching of repositories for an organization faster. Tested this code with an **organization that has around 3k repositories** ### Before (~2 Minutes) ``` TRC logger.FuncDur> func=provider.github.repositories took=102803.621667 ``` ### After (~5 seconds) ``` TRC logger.FuncDur> func=provider.github.repositories took=4567.576542 ``` * :zap: fetch org repositories in parallel * ⚙️ add a collector to the workerpool This will help us submit as many requests as we want without knowing about the workers. * :rotating_light: fix race conditions --------- Signed-off-by: Salim Afiune Maya --- internal/workerpool/collector.go | 55 ++++++ internal/workerpool/pool.go | 112 +++++++++++ internal/workerpool/pool_test.go | 185 ++++++++++++++++++ internal/workerpool/worker.go | 30 +++ providers-sdk/v1/inventory/inventory.pb.go | 4 +- providers-sdk/v1/plugin/plugin.pb.go | 4 +- providers-sdk/v1/plugin/plugin_grpc.pb.go | 46 ++--- providers-sdk/v1/resources/resources.pb.go | 4 +- providers-sdk/v1/vault/vault.pb.go | 4 +- providers/github/provider/provider.go | 3 +- providers/github/resources/github.go | 1 + providers/github/resources/github.lr | 2 + providers/github/resources/github.lr.go | 12 ++ .../github/resources/github.lr.manifest.yaml | 2 + providers/github/resources/github_org.go | 67 +++++-- 15 files changed, 469 insertions(+), 62 deletions(-) create mode 100644 internal/workerpool/collector.go create mode 100644 internal/workerpool/pool.go create mode 100644 internal/workerpool/pool_test.go create mode 100644 internal/workerpool/worker.go diff --git a/internal/workerpool/collector.go b/internal/workerpool/collector.go new file mode 100644 index 0000000000..2d105501be --- /dev/null +++ b/internal/workerpool/collector.go @@ -0,0 +1,55 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package workerpool + +import ( + "sync" + "sync/atomic" +) + +type collector[R any] struct { + resultsCh <-chan R + results []R + read sync.Mutex + + errorsCh <-chan error + errors []error + + requestsRead int64 +} + +func (c *collector[R]) start() { + go func() { + for { + select { + case result := <-c.resultsCh: + c.read.Lock() + c.results = append(c.results, result) + c.read.Unlock() + + case err := <-c.errorsCh: + c.read.Lock() + c.errors = append(c.errors, err) + c.read.Unlock() + } + + atomic.AddInt64(&c.requestsRead, 1) + } + }() +} +func (c *collector[R]) GetResults() []R { + c.read.Lock() + defer c.read.Unlock() + return c.results +} + +func (c *collector[R]) GetErrors() []error { + c.read.Lock() + defer c.read.Unlock() + return c.errors +} + +func (c *collector[R]) RequestsRead() int64 { + return atomic.LoadInt64(&c.requestsRead) +} diff --git a/internal/workerpool/pool.go b/internal/workerpool/pool.go new file mode 100644 index 0000000000..8553ca25d5 --- /dev/null +++ b/internal/workerpool/pool.go @@ -0,0 +1,112 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package workerpool + +import ( + "sync" + "sync/atomic" + "time" + + "github.com/cockroachdb/errors" +) + +type Task[R any] func() (result R, err error) + +// Pool is a generic pool of workers. +type Pool[R any] struct { + queueCh chan Task[R] + resultsCh chan R + errorsCh chan error + + requestsSent int64 + once sync.Once + + workers []*worker[R] + workerCount int + + collector[R] +} + +// New initializes a new Pool with the provided number of workers. The pool is generic and can +// accept any type of Task that returns the signature `func() (R, error)`. +// +// For example, a Pool[int] will accept Tasks similar to: +// +// task := func() (int, error) { +// return 42, nil +// } +func New[R any](count int) *Pool[R] { + resultsCh := make(chan R) + errorsCh := make(chan error) + return &Pool[R]{ + queueCh: make(chan Task[R]), + resultsCh: resultsCh, + errorsCh: errorsCh, + workerCount: count, + collector: collector[R]{resultsCh: resultsCh, errorsCh: errorsCh}, + } +} + +// Start the pool workers and collector. Make sure call `Close()` to clear the pool. +// +// pool := workerpool.New[int](10) +// pool.Start() +// defer pool.Close() +func (p *Pool[R]) Start() { + p.once.Do(func() { + for i := 0; i < p.workerCount; i++ { + w := worker[R]{id: i, queueCh: p.queueCh, resultsCh: p.resultsCh, errorsCh: p.errorsCh} + w.start() + p.workers = append(p.workers, &w) + } + + p.collector.start() + }) +} + +// Submit sends a task to the workers +func (p *Pool[R]) Submit(t Task[R]) { + p.queueCh <- t + atomic.AddInt64(&p.requestsSent, 1) +} + +// GetErrors returns any error from a processed task +func (p *Pool[R]) GetErrors() error { + return errors.Join(p.collector.GetErrors()...) +} + +// GetResults returns the tasks results. +// +// It is recommended to call `Wait()` before reading the results. +func (p *Pool[R]) GetResults() []R { + return p.collector.GetResults() +} + +// Close waits for workers and collector to process all the requests, and then closes +// the task queue channel. After closing the pool, calling `Submit()` will panic. +func (p *Pool[R]) Close() { + p.Wait() + close(p.queueCh) +} + +// Wait waits until all tasks have been processed. +func (p *Pool[R]) Wait() { + ticker := time.NewTicker(100 * time.Millisecond) + for { + if !p.Processing() { + return + } + <-ticker.C + } +} + +// PendingRequests returns the number of pending requests. +func (p *Pool[R]) PendingRequests() int64 { + return atomic.LoadInt64(&p.requestsSent) - p.collector.RequestsRead() +} + +// Processing return true if tasks are being processed. +func (p *Pool[R]) Processing() bool { + return p.PendingRequests() != 0 +} diff --git a/internal/workerpool/pool_test.go b/internal/workerpool/pool_test.go new file mode 100644 index 0000000000..3b3946df1e --- /dev/null +++ b/internal/workerpool/pool_test.go @@ -0,0 +1,185 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package workerpool_test + +import ( + "errors" + "testing" + "time" + + "math/rand" + + "github.com/stretchr/testify/assert" + "go.mondoo.com/cnquery/v11/internal/workerpool" +) + +func TestPoolSubmitAndRetrieveResult(t *testing.T) { + pool := workerpool.New[int](2) + pool.Start() + defer pool.Close() + + task := func() (int, error) { + return 42, nil + } + + // no results + assert.Empty(t, pool.GetResults()) + + // submit a request + pool.Submit(task) + + // wait for the request to process + pool.Wait() + + // should have one result + results := pool.GetResults() + if assert.Len(t, results, 1) { + assert.Equal(t, 42, results[0]) + } + + // no errors + assert.Nil(t, pool.GetErrors()) +} + +func TestPoolHandleErrors(t *testing.T) { + pool := workerpool.New[int](5) + pool.Start() + defer pool.Close() + + // submit a task that will return an error + task := func() (int, error) { + return 0, errors.New("task error") + } + pool.Submit(task) + + // Wait for error collector to process + pool.Wait() + + err := pool.GetErrors() + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "task error") + } +} + +func TestPoolMultipleTasksWithErrors(t *testing.T) { + type test struct { + data int + } + pool := workerpool.New[*test](5) + pool.Start() + defer pool.Close() + + tasks := []workerpool.Task[*test]{ + func() (*test, error) { return &test{1}, nil }, + func() (*test, error) { return &test{2}, nil }, + func() (*test, error) { + return nil, errors.New("task error") + }, + func() (*test, error) { return &test{3}, nil }, + } + + for _, task := range tasks { + pool.Submit(task) + } + + // Wait for error collector to process + pool.Wait() + + results := pool.GetResults() + assert.ElementsMatch(t, []*test{&test{1}, &test{2}, &test{3}}, results) + err := pool.GetErrors() + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "task error") + } +} + +func TestPoolHandlesNilTasks(t *testing.T) { + pool := workerpool.New[int](2) + pool.Start() + defer pool.Close() + + var nilTask workerpool.Task[int] + pool.Submit(nilTask) + + pool.Wait() + + err := pool.GetErrors() + assert.NoError(t, err) +} + +func TestPoolProcessing(t *testing.T) { + pool := workerpool.New[int](2) + pool.Start() + defer pool.Close() + + task := func() (int, error) { + time.Sleep(50 * time.Millisecond) + return 10, nil + } + + pool.Submit(task) + + // should be processing + assert.True(t, pool.Processing()) + + // wait + pool.Wait() + + // read results + result := pool.GetResults() + assert.Equal(t, []int{10}, result) + + // should not longer be processing + assert.False(t, pool.Processing()) +} + +func TestPoolClosesGracefully(t *testing.T) { + pool := workerpool.New[int](1) + pool.Start() + + task := func() (int, error) { + time.Sleep(100 * time.Millisecond) + return 42, nil + } + + pool.Submit(task) + + pool.Close() + + // Ensure no panic occurs and channels are closed + assert.PanicsWithError(t, "send on closed channel", func() { + pool.Submit(task) + }) +} + +func TestPoolWithManyTasks(t *testing.T) { + // 30k requests with a pool of 100 workers + // should be around 15 seconds + requestCount := 30000 + pool := workerpool.New[int](100) + pool.Start() + defer pool.Close() + + task := func() (int, error) { + random := rand.Intn(100) + time.Sleep(time.Duration(random) * time.Millisecond) + return random, nil + } + + for i := 0; i < requestCount; i++ { + pool.Submit(task) + } + + // should be processing + assert.True(t, pool.Processing()) + + // wait + pool.Wait() + + // read results + assert.Equal(t, requestCount, len(pool.GetResults())) + + // should not longer be processing + assert.False(t, pool.Processing()) +} diff --git a/internal/workerpool/worker.go b/internal/workerpool/worker.go new file mode 100644 index 0000000000..77b5c81f15 --- /dev/null +++ b/internal/workerpool/worker.go @@ -0,0 +1,30 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package workerpool + +type worker[R any] struct { + id int + queueCh <-chan Task[R] + resultsCh chan<- R + errorsCh chan<- error +} + +func (w *worker[R]) start() { + go func() { + for task := range w.queueCh { + if task == nil { + // let the collector know we processed the request + w.errorsCh <- nil + continue + } + + data, err := task() + if err != nil { + w.errorsCh <- err + } else { + w.resultsCh <- data + } + } + }() +} diff --git a/providers-sdk/v1/inventory/inventory.pb.go b/providers-sdk/v1/inventory/inventory.pb.go index 07cf91ce9f..6ae36eef19 100644 --- a/providers-sdk/v1/inventory/inventory.pb.go +++ b/providers-sdk/v1/inventory/inventory.pb.go @@ -3,8 +3,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.28.3 +// protoc-gen-go v1.35.2 +// protoc v5.29.0 // source: inventory.proto package inventory diff --git a/providers-sdk/v1/plugin/plugin.pb.go b/providers-sdk/v1/plugin/plugin.pb.go index bc99be0316..844819aa20 100644 --- a/providers-sdk/v1/plugin/plugin.pb.go +++ b/providers-sdk/v1/plugin/plugin.pb.go @@ -3,8 +3,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.28.3 +// protoc-gen-go v1.35.2 +// protoc v5.29.0 // source: plugin.proto package plugin diff --git a/providers-sdk/v1/plugin/plugin_grpc.pb.go b/providers-sdk/v1/plugin/plugin_grpc.pb.go index 4d1d3d352d..81b221fc9e 100644 --- a/providers-sdk/v1/plugin/plugin_grpc.pb.go +++ b/providers-sdk/v1/plugin/plugin_grpc.pb.go @@ -3,8 +3,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.5.1 -// - protoc v5.28.3 +// - protoc-gen-go-grpc v1.4.0 +// - protoc v5.29.0 // source: plugin.proto package plugin @@ -18,8 +18,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.64.0 or later. -const _ = grpc.SupportPackageIsVersion9 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 const ( ProviderPlugin_Heartbeat_FullMethodName = "/cnquery.providers.v1.ProviderPlugin/Heartbeat" @@ -136,7 +136,7 @@ func (c *providerPluginClient) StoreData(ctx context.Context, in *StoreReq, opts // ProviderPluginServer is the server API for ProviderPlugin service. // All implementations must embed UnimplementedProviderPluginServer -// for forward compatibility. +// for forward compatibility type ProviderPluginServer interface { Heartbeat(context.Context, *HeartbeatReq) (*HeartbeatRes, error) ParseCLI(context.Context, *ParseCLIReq) (*ParseCLIRes, error) @@ -149,12 +149,9 @@ type ProviderPluginServer interface { mustEmbedUnimplementedProviderPluginServer() } -// UnimplementedProviderPluginServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedProviderPluginServer struct{} +// UnimplementedProviderPluginServer must be embedded to have forward compatible implementations. +type UnimplementedProviderPluginServer struct { +} func (UnimplementedProviderPluginServer) Heartbeat(context.Context, *HeartbeatReq) (*HeartbeatRes, error) { return nil, status.Errorf(codes.Unimplemented, "method Heartbeat not implemented") @@ -181,7 +178,6 @@ func (UnimplementedProviderPluginServer) StoreData(context.Context, *StoreReq) ( return nil, status.Errorf(codes.Unimplemented, "method StoreData not implemented") } func (UnimplementedProviderPluginServer) mustEmbedUnimplementedProviderPluginServer() {} -func (UnimplementedProviderPluginServer) testEmbeddedByValue() {} // UnsafeProviderPluginServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ProviderPluginServer will @@ -191,13 +187,6 @@ type UnsafeProviderPluginServer interface { } func RegisterProviderPluginServer(s grpc.ServiceRegistrar, srv ProviderPluginServer) { - // If the following call pancis, it indicates UnimplementedProviderPluginServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } s.RegisterService(&ProviderPlugin_ServiceDesc, srv) } @@ -444,7 +433,7 @@ func (c *providerCallbackClient) GetData(ctx context.Context, in *DataReq, opts // ProviderCallbackServer is the server API for ProviderCallback service. // All implementations must embed UnimplementedProviderCallbackServer -// for forward compatibility. +// for forward compatibility type ProviderCallbackServer interface { Collect(context.Context, *DataRes) (*CollectRes, error) GetRecording(context.Context, *DataReq) (*ResourceData, error) @@ -452,12 +441,9 @@ type ProviderCallbackServer interface { mustEmbedUnimplementedProviderCallbackServer() } -// UnimplementedProviderCallbackServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedProviderCallbackServer struct{} +// UnimplementedProviderCallbackServer must be embedded to have forward compatible implementations. +type UnimplementedProviderCallbackServer struct { +} func (UnimplementedProviderCallbackServer) Collect(context.Context, *DataRes) (*CollectRes, error) { return nil, status.Errorf(codes.Unimplemented, "method Collect not implemented") @@ -469,7 +455,6 @@ func (UnimplementedProviderCallbackServer) GetData(context.Context, *DataReq) (* return nil, status.Errorf(codes.Unimplemented, "method GetData not implemented") } func (UnimplementedProviderCallbackServer) mustEmbedUnimplementedProviderCallbackServer() {} -func (UnimplementedProviderCallbackServer) testEmbeddedByValue() {} // UnsafeProviderCallbackServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ProviderCallbackServer will @@ -479,13 +464,6 @@ type UnsafeProviderCallbackServer interface { } func RegisterProviderCallbackServer(s grpc.ServiceRegistrar, srv ProviderCallbackServer) { - // If the following call pancis, it indicates UnimplementedProviderCallbackServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } s.RegisterService(&ProviderCallback_ServiceDesc, srv) } diff --git a/providers-sdk/v1/resources/resources.pb.go b/providers-sdk/v1/resources/resources.pb.go index 35ba6cad44..19797b6200 100644 --- a/providers-sdk/v1/resources/resources.pb.go +++ b/providers-sdk/v1/resources/resources.pb.go @@ -3,8 +3,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.28.3 +// protoc-gen-go v1.35.2 +// protoc v5.29.0 // source: resources.proto package resources diff --git a/providers-sdk/v1/vault/vault.pb.go b/providers-sdk/v1/vault/vault.pb.go index cfb80dda7e..ad919c8cb4 100644 --- a/providers-sdk/v1/vault/vault.pb.go +++ b/providers-sdk/v1/vault/vault.pb.go @@ -3,8 +3,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.35.1 -// protoc v5.28.3 +// protoc-gen-go v1.35.2 +// protoc v5.29.0 // source: vault.proto package vault diff --git a/providers/github/provider/provider.go b/providers/github/provider/provider.go index 970dc541af..25cdfdeaa1 100644 --- a/providers/github/provider/provider.go +++ b/providers/github/provider/provider.go @@ -53,7 +53,8 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error) } isAppAuth := false - if appId, ok := req.Flags[connection.OPTION_APP_ID]; ok && len(appId.Value) > 0 { + appId, ok := flags[connection.OPTION_APP_ID] + if ok && len(appId.Value) > 0 { conf.Options[connection.OPTION_APP_ID] = string(appId.Value) installId := req.Flags[connection.OPTION_APP_INSTALLATION_ID] diff --git a/providers/github/resources/github.go b/providers/github/resources/github.go index 2510b176eb..bccbd372fd 100644 --- a/providers/github/resources/github.go +++ b/providers/github/resources/github.go @@ -77,4 +77,5 @@ func githubTimestamp(ts *github.Timestamp) *time.Time { const ( paginationPerPage = 100 + workers = 10 ) diff --git a/providers/github/resources/github.lr b/providers/github/resources/github.lr index b3b3a6b3cb..67bb1f8e6e 100644 --- a/providers/github/resources/github.lr +++ b/providers/github/resources/github.lr @@ -80,6 +80,8 @@ github.organization @defaults("login name") { updatedAt time // Number of private repositories totalPrivateRepos int + // Number of public repositories + totalPublicRepos int // Number of owned private repositories for the organization ownedPrivateRepos int // Number of private gists diff --git a/providers/github/resources/github.lr.go b/providers/github/resources/github.lr.go index beb8b4516f..44725e97b2 100644 --- a/providers/github/resources/github.lr.go +++ b/providers/github/resources/github.lr.go @@ -276,6 +276,9 @@ var getDataFields = map[string]func(r plugin.Resource) *plugin.DataRes{ "github.organization.totalPrivateRepos": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubOrganization).GetTotalPrivateRepos()).ToDataRes(types.Int) }, + "github.organization.totalPublicRepos": func(r plugin.Resource) *plugin.DataRes { + return (r.(*mqlGithubOrganization).GetTotalPublicRepos()).ToDataRes(types.Int) + }, "github.organization.ownedPrivateRepos": func(r plugin.Resource) *plugin.DataRes { return (r.(*mqlGithubOrganization).GetOwnedPrivateRepos()).ToDataRes(types.Int) }, @@ -1118,6 +1121,10 @@ var setDataFields = map[string]func(r plugin.Resource, v *llx.RawData) bool { r.(*mqlGithubOrganization).TotalPrivateRepos, ok = plugin.RawToTValue[int64](v.Value, v.Error) return }, + "github.organization.totalPublicRepos": func(r plugin.Resource, v *llx.RawData) (ok bool) { + r.(*mqlGithubOrganization).TotalPublicRepos, ok = plugin.RawToTValue[int64](v.Value, v.Error) + return + }, "github.organization.ownedPrivateRepos": func(r plugin.Resource, v *llx.RawData) (ok bool) { r.(*mqlGithubOrganization).OwnedPrivateRepos, ok = plugin.RawToTValue[int64](v.Value, v.Error) return @@ -2404,6 +2411,7 @@ type mqlGithubOrganization struct { CreatedAt plugin.TValue[*time.Time] UpdatedAt plugin.TValue[*time.Time] TotalPrivateRepos plugin.TValue[int64] + TotalPublicRepos plugin.TValue[int64] OwnedPrivateRepos plugin.TValue[int64] PrivateGists plugin.TValue[int64] DiskUsage plugin.TValue[int64] @@ -2533,6 +2541,10 @@ func (c *mqlGithubOrganization) GetTotalPrivateRepos() *plugin.TValue[int64] { return &c.TotalPrivateRepos } +func (c *mqlGithubOrganization) GetTotalPublicRepos() *plugin.TValue[int64] { + return &c.TotalPublicRepos +} + func (c *mqlGithubOrganization) GetOwnedPrivateRepos() *plugin.TValue[int64] { return &c.OwnedPrivateRepos } diff --git a/providers/github/resources/github.lr.manifest.yaml b/providers/github/resources/github.lr.manifest.yaml index a5f4adf8dd..af798f362b 100755 --- a/providers/github/resources/github.lr.manifest.yaml +++ b/providers/github/resources/github.lr.manifest.yaml @@ -274,6 +274,8 @@ resources: total_private_repos: {} totalPrivateRepos: min_mondoo_version: 6.11.0 + totalPublicRepos: + min_mondoo_version: 9.0.0 twitter_username: {} twitterUsername: min_mondoo_version: 6.11.0 diff --git a/providers/github/resources/github_org.go b/providers/github/resources/github_org.go index 67005eb7ab..ef39f97159 100644 --- a/providers/github/resources/github_org.go +++ b/providers/github/resources/github_org.go @@ -5,11 +5,14 @@ package resources import ( "errors" + "slices" "strconv" "strings" "time" "github.com/google/go-github/v67/github" + "github.com/rs/zerolog/log" + "go.mondoo.com/cnquery/v11/internal/workerpool" "go.mondoo.com/cnquery/v11/llx" "go.mondoo.com/cnquery/v11/logger" "go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin" @@ -70,6 +73,7 @@ func initGithubOrganization(runtime *plugin.Runtime, args map[string]*llx.RawDat args["createdAt"] = llx.TimeDataPtr(githubTimestamp(org.CreatedAt)) args["updatedAt"] = llx.TimeDataPtr(githubTimestamp(org.UpdatedAt)) args["totalPrivateRepos"] = llx.IntDataPtr(org.TotalPrivateRepos) + args["totalPublicRepos"] = llx.IntDataPtr(org.PublicRepos) args["ownedPrivateRepos"] = llx.IntDataPtr(org.OwnedPrivateRepos) args["privateGists"] = llx.IntDataDefault(org.PrivateGists, 0) args["diskUsage"] = llx.IntDataDefault(org.DiskUsage, 0) @@ -262,26 +266,49 @@ func (g *mqlGithubOrganization) repositories() ([]interface{}, error) { return nil, g.Login.Error } orgLogin := g.Login.Data - - listOpts := &github.RepositoryListByOrgOptions{ - ListOptions: github.ListOptions{PerPage: paginationPerPage}, - Type: "all", + listOpts := github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{ + PerPage: paginationPerPage, + Page: 1, + }, + Type: "all", } - var allRepos []*github.Repository + repoCount := g.TotalPrivateRepos.Data + g.TotalPublicRepos.Data + workerPool := workerpool.New[[]*github.Repository](workers) + workerPool.Start() + defer workerPool.Close() + + log.Debug(). + Int("workers", workers). + Int64("total_repos", repoCount). + Str("organization", g.Name.Data). + Msg("list repositories") + for { - repos, resp, err := conn.Client().Repositories.ListByOrg(conn.Context(), orgLogin, listOpts) - if err != nil { + // exit as soon as we collect all repositories + reposLen := len(slices.Concat(workerPool.GetResults()...)) + if reposLen >= int(repoCount) { + break + } + + // send requests to workers + opts := listOpts + workerPool.Submit(func() ([]*github.Repository, error) { + repos, _, err := conn.Client().Repositories.ListByOrg(conn.Context(), orgLogin, &opts) + return repos, err + }) + + // next page + listOpts.Page++ + + // check if any request failed + if err := workerPool.GetErrors(); err != nil { if strings.Contains(err.Error(), "404") { return nil, nil } return nil, err } - allRepos = append(allRepos, repos...) - if resp.NextPage == 0 { - break - } - listOpts.Page = resp.NextPage } if g.repoCacheMap == nil { @@ -289,15 +316,17 @@ func (g *mqlGithubOrganization) repositories() ([]interface{}, error) { } res := []interface{}{} - for i := range allRepos { - repo := allRepos[i] + for _, repos := range workerPool.GetResults() { + for i := range repos { + repo := repos[i] - r, err := newMqlGithubRepository(g.MqlRuntime, repo) - if err != nil { - return nil, err + r, err := newMqlGithubRepository(g.MqlRuntime, repo) + if err != nil { + return nil, err + } + res = append(res, r) + g.repoCacheMap[repo.GetName()] = r } - res = append(res, r) - g.repoCacheMap[repo.GetName()] = r } return res, nil