From 7efebde452128dbd63d83c2ff2565f8a69d0f279 Mon Sep 17 00:00:00 2001 From: vjeffrey Date: Mon, 9 Oct 2023 02:51:41 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix=20gitlab=20discovery=20for?= =?UTF-8?q?=20enterprise=20gitlab=20(#2124)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * WIP fix gitlab discovery for enterprise gitlab * fix repo parsing for gitlab with subgroups Signed-off-by: Ivan Milchev * fix go.mod Signed-off-by: Ivan Milchev --------- Signed-off-by: Ivan Milchev Co-authored-by: Ivan Milchev --- providers/gitlab/connection/connection.go | 94 +++++++++++++++++++ providers/gitlab/provider/discovery.go | 94 +++++-------------- providers/gitlab/provider/provider.go | 19 ++-- providers/network/resources/dns_test.go | 10 +- providers/terraform/provider/detector.go | 8 +- providers/terraform/provider/detector_test.go | 9 ++ .../resources/terraform.lr.manifest.yaml | 8 +- 7 files changed, 150 insertions(+), 92 deletions(-) diff --git a/providers/gitlab/connection/connection.go b/providers/gitlab/connection/connection.go index 1893e93c20..ed5dbe1a74 100644 --- a/providers/gitlab/connection/connection.go +++ b/providers/gitlab/connection/connection.go @@ -7,6 +7,8 @@ import ( "errors" "net/url" "os" + "strconv" + "strings" "github.com/rs/zerolog/log" "github.com/xanzy/go-gitlab" @@ -22,8 +24,10 @@ type GitLabConnection struct { project *gitlab.Project projectID string // only used for initial setup, use project.ID afterwards! groupName string + groupID string projectName string client *gitlab.Client + url string } func NewGitLabConnection(id uint32, asset *inventory.Asset, conf *inventory.Config) (*GitLabConnection, error) { @@ -67,8 +71,10 @@ func NewGitLabConnection(id uint32, asset *inventory.Asset, conf *inventory.Conf id: id, asset: asset, groupName: conf.Options["group"], + groupID: conf.Options["group-id"], projectName: conf.Options["project"], projectID: conf.Options["project-id"], + url: conf.Options["url"], client: client, } @@ -87,6 +93,18 @@ func (c *GitLabConnection) Asset() *inventory.Asset { return c.asset } +func (c *GitLabConnection) GroupName() string { + return c.groupName +} + +func (c *GitLabConnection) GroupID() int { + i, err := strconv.Atoi(c.groupID) + if err == nil { + return i + } + return 0 +} + func (c *GitLabConnection) Client() *gitlab.Client { return c.client } @@ -99,11 +117,30 @@ func (c *GitLabConnection) Group() (*gitlab.Group, error) { return nil, errors.New("cannot look up gitlab group, no group name defined") } + // if group name has a slash, we know its a subgroup + if names := strings.Split(c.groupName, "/"); len(names) > 1 { + return c.findSubgroup(names[0], names[1]) + } + var err error c.group, _, err = c.Client().Groups.GetGroup(c.groupName, nil) return c.group, err } +func (c *GitLabConnection) findSubgroup(parentId string, name string) (*gitlab.Group, error) { + log.Debug().Msgf("find subgroup for %s %s", parentId, name) + groups, err := DiscoverSubAndDescendantGroupsForGroup(c, parentId) + if err != nil { + return nil, err + } + for i := range groups { + if name == groups[i].Name { + return groups[i], nil + } + } + return nil, errors.New("not found") +} + func (c *GitLabConnection) IsGroup() bool { return c.groupName != "" } @@ -145,3 +182,60 @@ func (c *GitLabConnection) Project() (*gitlab.Project, error) { c.project, _, err = c.Client().Projects.GetProject(pid, nil) return c.project, err } + +func DiscoverSubAndDescendantGroupsForGroup(conn *GitLabConnection, rootGroup string) ([]*gitlab.Group, error) { + var list []*gitlab.Group + // discover subgroups + subgroups, err := groupSubgroups(conn, rootGroup) + if err != nil { + log.Debug().Err(err).Msgf("cannot discover subgroups for %v", rootGroup) + } else { + list = append(list, subgroups...) + } + // discover descendant groups + descgroups, err := groupDescendantGroups(conn, rootGroup) + if err != nil { + log.Debug().Err(err).Msgf("cannot discover descendant groups for %v", rootGroup) + } else { + list = append(list, descgroups...) + } + return list, nil +} + +func groupDescendantGroups(conn *GitLabConnection, gid interface{}) ([]*gitlab.Group, error) { + log.Debug().Msgf("calling list descendant groups with %v", gid) + perPage := 50 + page := 1 + total := 50 + groups := []*gitlab.Group{} + for page*perPage <= total { + grps, resp, err := conn.Client().Groups.ListDescendantGroups(gid, &gitlab.ListDescendantGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}}) + if err != nil { + return nil, err + } + groups = append(groups, grps...) + total = resp.TotalItems + page += 1 + } + + return groups, nil +} + +func groupSubgroups(conn *GitLabConnection, gid interface{}) ([]*gitlab.Group, error) { + log.Debug().Msgf("calling list subgroups with %v", gid) + perPage := 50 + page := 1 + total := 50 + groups := []*gitlab.Group{} + for page*perPage <= total { + grps, resp, err := conn.Client().Groups.ListSubGroups(gid, &gitlab.ListSubGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}}) + if err != nil { + return nil, err + } + groups = append(groups, grps...) + total = resp.TotalItems + page += 1 + } + + return groups, nil +} diff --git a/providers/gitlab/provider/discovery.go b/providers/gitlab/provider/discovery.go index 3b852f44a5..b9aa8cb339 100644 --- a/providers/gitlab/provider/discovery.go +++ b/providers/gitlab/provider/discovery.go @@ -78,7 +78,7 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC if err != nil { return nil, nil, err } - return s.convertGitlabGroupsToAssetGroups(groups, conn), groups, nil + return s.convertGitlabGroupsToAssetGroups(groups, conn, ""), groups, nil } if conn.IsGroup() { @@ -88,14 +88,18 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC } groups := []*gitlab.Group{group} assets := []*inventory.Asset{root} + if names := strings.Split(group.Name, "/"); len(names) > 1 { + log.Debug().Msg("skipping subgroup discovery for subgroup") + return assets, groups, nil + } // discover subgroups and descendant groups - subgroups, err := discoverSubAndDescendantGroupsForGroup(conn) + subgroups, err := connection.DiscoverSubAndDescendantGroupsForGroup(conn, group.Path) if err != nil { log.Error().Err(err).Msg("unable to discover sub groups") return []*inventory.Asset{root}, []*gitlab.Group{group}, err } groups = append(groups, subgroups...) - assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn)...) + assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn, group.Path)...) return assets, groups, err } @@ -107,8 +111,9 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC conf := conn.Conf.Clone() conf.Type = GitlabGroupConnection conf.Options = map[string]string{ - "group": group.Name, + "group": group.FullPath, "group-id": strconv.Itoa(group.ID), + "url": conn.Conf.Options["url"], } asset := &inventory.Asset{ Connections: []*inventory.Config{conf}, @@ -118,18 +123,23 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC groups := []*gitlab.Group{group} assets := []*inventory.Asset{asset} + if names := strings.Split(group.Name, "/"); len(names) > 1 { + log.Debug().Msg("skipping subgroup discovery for subgroup") + return assets, groups, nil + } // discover subgroups and descendant groups - subgroups, err := discoverSubAndDescendantGroupsForGroup(conn) + subgroups, err := connection.DiscoverSubAndDescendantGroupsForGroup(conn, group.Path) if err != nil { log.Error().Err(err).Msg("unable to discover sub groups") return []*inventory.Asset{root}, []*gitlab.Group{group}, err } groups = append(groups, subgroups...) - assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn)...) + assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn, group.Path)...) return assets, groups, nil } func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLabConnection, groups []*gitlab.Group) ([]*inventory.Asset, []*gitlab.Project, error) { + log.Debug().Msg("discover projects") if conn.IsProject() { project, err := conn.Project() return []*inventory.Asset{root}, []*gitlab.Project{project}, err @@ -140,7 +150,7 @@ func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLa for i := range groups { group := groups[i] - groupProjects, err := discoverGroupProjects(conn, group.ID) + groupProjects, err := discoverGroupProjects(conn, group.FullPath) if err != nil { return nil, nil, err } @@ -150,17 +160,18 @@ func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLa conf := conn.Conf.Clone() conf.Type = GitlabProjectConnection conf.Options = map[string]string{ - "group": group.Name, + "group": group.FullPath, "group-id": strconv.Itoa(group.ID), "project": project.Name, "project-id": strconv.Itoa(project.ID), + "url": conn.Conf.Options["url"], } asset := &inventory.Asset{ Name: project.NameWithNamespace, Connections: []*inventory.Config{conf}, } - s.detectAsProject(asset, group, project) + s.detectAsProject(asset, group.ID, group.FullPath, project) if err != nil { return nil, nil, err } @@ -173,6 +184,7 @@ func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLa } func discoverGroupProjects(conn *connection.GitLabConnection, gid interface{}) ([]*gitlab.Project, error) { + log.Debug().Msgf("discover group projects for %v", gid) perPage := 50 page := 1 total := 50 @@ -190,7 +202,7 @@ func discoverGroupProjects(conn *connection.GitLabConnection, gid interface{}) ( return projects, nil } -func (s *Service) convertGitlabGroupsToAssetGroups(groups []*gitlab.Group, conn *connection.GitLabConnection) []*inventory.Asset { +func (s *Service) convertGitlabGroupsToAssetGroups(groups []*gitlab.Group, conn *connection.GitLabConnection, rootGroupPath string) []*inventory.Asset { var list []*inventory.Asset // convert to assets for _, group := range groups { @@ -198,7 +210,9 @@ func (s *Service) convertGitlabGroupsToAssetGroups(groups []*gitlab.Group, conn if conf.Options == nil { conf.Options = map[string]string{} } - conf.Options["group"] = group.Path + conf.Options["group"] = group.FullPath + conf.Options["group-id"] = strconv.Itoa(group.ID) + conf.Options["url"] = conn.Conf.Options["url"] conf.Type = GitlabGroupConnection asset := &inventory.Asset{ Connections: []*inventory.Config{conf}, @@ -213,64 +227,8 @@ func (s *Service) convertGitlabGroupsToAssetGroups(groups []*gitlab.Group, conn return list } -func discoverSubAndDescendantGroupsForGroup(conn *connection.GitLabConnection) ([]*gitlab.Group, error) { - gid, err := conn.GID() - if err != nil { - return nil, err - } - var list []*gitlab.Group - // discover subgroups - subgroups, err := groupSubgroups(conn, gid) - if err != nil { - return nil, err - } - list = append(list, subgroups...) - // discover descendant groups - descgroups, err := groupDescendantGroups(conn, gid) - if err != nil { - return nil, err - } - list = append(list, descgroups...) - return list, nil -} - -func groupDescendantGroups(conn *connection.GitLabConnection, gid interface{}) ([]*gitlab.Group, error) { - perPage := 50 - page := 1 - total := 50 - groups := []*gitlab.Group{} - for page*perPage <= total { - grps, resp, err := conn.Client().Groups.ListDescendantGroups(gid, &gitlab.ListDescendantGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}}) - if err != nil { - return nil, err - } - groups = append(groups, grps...) - total = resp.TotalItems - page += 1 - } - - return groups, nil -} - -func groupSubgroups(conn *connection.GitLabConnection, gid interface{}) ([]*gitlab.Group, error) { - perPage := 50 - page := 1 - total := 50 - groups := []*gitlab.Group{} - for page*perPage <= total { - grps, resp, err := conn.Client().Groups.ListSubGroups(gid, &gitlab.ListSubGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}}) - if err != nil { - return nil, err - } - groups = append(groups, grps...) - total = resp.TotalItems - page += 1 - } - - return groups, nil -} - func listAllGroups(conn *connection.GitLabConnection) ([]*gitlab.Group, error) { + log.Debug().Msg("calling list all groups") perPage := 50 page := 1 total := 50 diff --git a/providers/gitlab/provider/provider.go b/providers/gitlab/provider/provider.go index 1f1ef41b05..0747b70520 100644 --- a/providers/gitlab/provider/provider.go +++ b/providers/gitlab/provider/provider.go @@ -225,30 +225,29 @@ func (s *Service) detect(asset *inventory.Asset, conn *connection.GitLabConnecti // we will discover the groups return nil } - group, err := conn.Group() - if err != nil { - return err - } if conn.IsProject() { project, err := conn.Project() if err != nil { return err } - s.detectAsProject(asset, group, project) - + s.detectAsProject(asset, conn.GroupID(), conn.GroupName(), project) // TODO fix 0 } else { + group, err := conn.Group() + if err != nil { + return err + } s.detectAsGroup(asset, group) } return nil } -func (s *Service) detectAsProject(asset *inventory.Asset, group *gitlab.Group, project *gitlab.Project) { +func (s *Service) detectAsProject(asset *inventory.Asset, groupID int, groupFullPath string, project *gitlab.Project) { asset.Platform = projectPlatform asset.Name = "GitLab Project " + project.Name asset.PlatformIds = []string{ - newGitLabProjectID(group.ID, project.ID), - newGitLabProjectIDFromPaths(group.Path, project.Path), // for backwards compatibility with v8 + newGitLabProjectID(groupID, project.ID), + newGitLabProjectIDFromPaths(groupFullPath, project.Path), // for backwards compatibility with v8 } } @@ -257,7 +256,7 @@ func (s *Service) detectAsGroup(asset *inventory.Asset, group *gitlab.Group) err asset.Name = "GitLab Group " + group.Name asset.PlatformIds = []string{ newGitLabGroupID(group.ID), - newGitLabGroupIDFromPath(group.Path), // for backwards compatibility with v8 + newGitLabGroupIDFromPath(group.FullPath), // for backwards compatibility with v8 } return nil } diff --git a/providers/network/resources/dns_test.go b/providers/network/resources/dns_test.go index bb7824d995..550faed6b1 100644 --- a/providers/network/resources/dns_test.go +++ b/providers/network/resources/dns_test.go @@ -8,11 +8,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.mondoo.com/cnquery/llx" - "go.mondoo.com/cnquery/providers-sdk/v1/inventory" - "go.mondoo.com/cnquery/providers-sdk/v1/plugin" - "go.mondoo.com/cnquery/providers/network/connection" - "go.mondoo.com/cnquery/providers/network/resources" + "go.mondoo.com/cnquery/v9/llx" + "go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory" + "go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin" + "go.mondoo.com/cnquery/v9/providers/network/connection" + "go.mondoo.com/cnquery/v9/providers/network/resources" ) func TestResource_DNS(t *testing.T) { diff --git a/providers/terraform/provider/detector.go b/providers/terraform/provider/detector.go index 35e834a3a1..ce931ad2f0 100644 --- a/providers/terraform/provider/detector.go +++ b/providers/terraform/provider/detector.go @@ -127,12 +127,10 @@ func parseSSHURL(url string) (string, string, string, error) { // Now split the second part at the slash to separate the org and repo orgRepoParts := strings.Split(providerParts[1], "/") - if len(orgRepoParts) != 2 { - return "", "", "", fmt.Errorf("malformed URL") - } - // The repo name includes .git, so we remove that - repo := strings.TrimSuffix(orgRepoParts[1], ".git") + // The repo name is the last part after the split. It includes .git, + // so we remove that + repo := strings.TrimSuffix(orgRepoParts[len(orgRepoParts)-1], ".git") return provider, orgRepoParts[0], repo, nil } diff --git a/providers/terraform/provider/detector_test.go b/providers/terraform/provider/detector_test.go index bc10b17fbc..4f3674b610 100644 --- a/providers/terraform/provider/detector_test.go +++ b/providers/terraform/provider/detector_test.go @@ -23,3 +23,12 @@ func TestDetectNameFromSsh(t *testing.T) { assert.Equal(t, "exampleorg", org) assert.Equal(t, "example-gitlab", repo) } + +func TestDetectNameFromSsh_GitlabSubgroups(t *testing.T) { + url := "git@gitlab.example.com:exampleorg/group/example-gitlab.git" + domain, org, repo, err := parseSSHURL(url) + require.NoError(t, err) + assert.Equal(t, "gitlab.example.com", domain) + assert.Equal(t, "exampleorg", org) + assert.Equal(t, "example-gitlab", repo) +} diff --git a/providers/terraform/resources/terraform.lr.manifest.yaml b/providers/terraform/resources/terraform.lr.manifest.yaml index c2f77bd338..1480fe773e 100755 --- a/providers/terraform/resources/terraform.lr.manifest.yaml +++ b/providers/terraform/resources/terraform.lr.manifest.yaml @@ -99,7 +99,7 @@ resources: min_mondoo_version: 8.10.0 platform: name: - - terraform-hcl + - terraform-plan terraform.plan.proposedChange: fields: actions: {} @@ -129,7 +129,7 @@ resources: min_mondoo_version: latest platform: name: - - terraform-hcl + - terraform-plan terraform.settings: fields: backend: @@ -161,7 +161,7 @@ resources: min_mondoo_version: 6.11.0 platform: name: - - terraform-state + - terraform-hcl terraform.state.output: fields: identifier: {} @@ -187,4 +187,4 @@ resources: min_mondoo_version: 6.11.0 platform: name: - - terraform-hcl + - terraform-state