Skip to content

Commit

Permalink
🐛 fix gitlab discovery for enterprise gitlab (#2124)
Browse files Browse the repository at this point in the history 
* WIP fix gitlab discovery for enterprise gitlab

* fix repo parsing for gitlab with subgroups

Signed-off-by: Ivan Milchev <[email protected]>

* fix go.mod

Signed-off-by: Ivan Milchev <[email protected]>

---------

Signed-off-by: Ivan Milchev <[email protected]>
Co-authored-by: Ivan Milchev <[email protected]>
  • Loading branch information
@vjeffrey @imilchev
vjeffrey and imilchev authored Oct 9, 2023
1 parent 186149b commit 7efebde
Show file tree
Hide file tree
Showing 7 changed files with 150 additions and 92 deletions.
94 changes: 94 additions & 0 deletions providers/gitlab/connection/connection.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ import (
"errors"
"net/url"
"os"
"strconv"
"strings"

"github.com/rs/zerolog/log"
"github.com/xanzy/go-gitlab"
Expand All @@ -22,8 +24,10 @@ type GitLabConnection struct {
project *gitlab.Project
projectID string // only used for initial setup, use project.ID afterwards!
groupName string
groupID string
projectName string
client *gitlab.Client
url string
}

func NewGitLabConnection(id uint32, asset *inventory.Asset, conf *inventory.Config) (*GitLabConnection, error) {
Expand Down Expand Up @@ -67,8 +71,10 @@ func NewGitLabConnection(id uint32, asset *inventory.Asset, conf *inventory.Conf
id: id,
asset: asset,
groupName: conf.Options["group"],
groupID: conf.Options["group-id"],
projectName: conf.Options["project"],
projectID: conf.Options["project-id"],
url: conf.Options["url"],
client: client,
}

Expand All @@ -87,6 +93,18 @@ func (c *GitLabConnection) Asset() *inventory.Asset {
return c.asset
}

func (c *GitLabConnection) GroupName() string {
return c.groupName
}

func (c *GitLabConnection) GroupID() int {
i, err := strconv.Atoi(c.groupID)
if err == nil {
return i
}
return 0
}

func (c *GitLabConnection) Client() *gitlab.Client {
return c.client
}
Expand All @@ -99,11 +117,30 @@ func (c *GitLabConnection) Group() (*gitlab.Group, error) {
return nil, errors.New("cannot look up gitlab group, no group name defined")
}

// if group name has a slash, we know its a subgroup
if names := strings.Split(c.groupName, "/"); len(names) > 1 {
return c.findSubgroup(names[0], names[1])
}

var err error
c.group, _, err = c.Client().Groups.GetGroup(c.groupName, nil)
return c.group, err
}

func (c *GitLabConnection) findSubgroup(parentId string, name string) (*gitlab.Group, error) {
log.Debug().Msgf("find subgroup for %s %s", parentId, name)
groups, err := DiscoverSubAndDescendantGroupsForGroup(c, parentId)
if err != nil {
return nil, err
}
for i := range groups {
if name == groups[i].Name {
return groups[i], nil
}
}
return nil, errors.New("not found")
}

func (c *GitLabConnection) IsGroup() bool {
return c.groupName != ""
}
Expand Down Expand Up @@ -145,3 +182,60 @@ func (c *GitLabConnection) Project() (*gitlab.Project, error) {
c.project, _, err = c.Client().Projects.GetProject(pid, nil)
return c.project, err
}

func DiscoverSubAndDescendantGroupsForGroup(conn *GitLabConnection, rootGroup string) ([]*gitlab.Group, error) {
var list []*gitlab.Group
// discover subgroups
subgroups, err := groupSubgroups(conn, rootGroup)
if err != nil {
log.Debug().Err(err).Msgf("cannot discover subgroups for %v", rootGroup)
} else {
list = append(list, subgroups...)
}
// discover descendant groups
descgroups, err := groupDescendantGroups(conn, rootGroup)
if err != nil {
log.Debug().Err(err).Msgf("cannot discover descendant groups for %v", rootGroup)
} else {
list = append(list, descgroups...)
}
return list, nil
}

func groupDescendantGroups(conn *GitLabConnection, gid interface{}) ([]*gitlab.Group, error) {
log.Debug().Msgf("calling list descendant groups with %v", gid)
perPage := 50
page := 1
total := 50
groups := []*gitlab.Group{}
for page*perPage <= total {
grps, resp, err := conn.Client().Groups.ListDescendantGroups(gid, &gitlab.ListDescendantGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}})
if err != nil {
return nil, err
}
groups = append(groups, grps...)
total = resp.TotalItems
page += 1
}

return groups, nil
}

func groupSubgroups(conn *GitLabConnection, gid interface{}) ([]*gitlab.Group, error) {
log.Debug().Msgf("calling list subgroups with %v", gid)
perPage := 50
page := 1
total := 50
groups := []*gitlab.Group{}
for page*perPage <= total {
grps, resp, err := conn.Client().Groups.ListSubGroups(gid, &gitlab.ListSubGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}})
if err != nil {
return nil, err
}
groups = append(groups, grps...)
total = resp.TotalItems
page += 1
}

return groups, nil
}
94 changes: 26 additions & 68 deletions providers/gitlab/provider/discovery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC
if err != nil {
return nil, nil, err
}
return s.convertGitlabGroupsToAssetGroups(groups, conn), groups, nil
return s.convertGitlabGroupsToAssetGroups(groups, conn, ""), groups, nil
}

if conn.IsGroup() {
Expand All @@ -88,14 +88,18 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC
}
groups := []*gitlab.Group{group}
assets := []*inventory.Asset{root}
if names := strings.Split(group.Name, "/"); len(names) > 1 {
log.Debug().Msg("skipping subgroup discovery for subgroup")
return assets, groups, nil
}
// discover subgroups and descendant groups
subgroups, err := discoverSubAndDescendantGroupsForGroup(conn)
subgroups, err := connection.DiscoverSubAndDescendantGroupsForGroup(conn, group.Path)
if err != nil {
log.Error().Err(err).Msg("unable to discover sub groups")
return []*inventory.Asset{root}, []*gitlab.Group{group}, err
}
groups = append(groups, subgroups...)
assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn)...)
assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn, group.Path)...)
return assets, groups, err
}

Expand All @@ -107,8 +111,9 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC
conf := conn.Conf.Clone()
conf.Type = GitlabGroupConnection
conf.Options = map[string]string{
"group": group.Name,
"group": group.FullPath,
"group-id": strconv.Itoa(group.ID),
"url": conn.Conf.Options["url"],
}
asset := &inventory.Asset{
Connections: []*inventory.Config{conf},
Expand All @@ -118,18 +123,23 @@ func (s *Service) discoverGroups(root *inventory.Asset, conn *connection.GitLabC

groups := []*gitlab.Group{group}
assets := []*inventory.Asset{asset}
if names := strings.Split(group.Name, "/"); len(names) > 1 {
log.Debug().Msg("skipping subgroup discovery for subgroup")
return assets, groups, nil
}
// discover subgroups and descendant groups
subgroups, err := discoverSubAndDescendantGroupsForGroup(conn)
subgroups, err := connection.DiscoverSubAndDescendantGroupsForGroup(conn, group.Path)
if err != nil {
log.Error().Err(err).Msg("unable to discover sub groups")
return []*inventory.Asset{root}, []*gitlab.Group{group}, err
}
groups = append(groups, subgroups...)
assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn)...)
assets = append(assets, s.convertGitlabGroupsToAssetGroups(subgroups, conn, group.Path)...)
return assets, groups, nil
}

func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLabConnection, groups []*gitlab.Group) ([]*inventory.Asset, []*gitlab.Project, error) {
log.Debug().Msg("discover projects")
if conn.IsProject() {
project, err := conn.Project()
return []*inventory.Asset{root}, []*gitlab.Project{project}, err
Expand All @@ -140,7 +150,7 @@ func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLa

for i := range groups {
group := groups[i]
groupProjects, err := discoverGroupProjects(conn, group.ID)
groupProjects, err := discoverGroupProjects(conn, group.FullPath)
if err != nil {
return nil, nil, err
}
Expand All @@ -150,17 +160,18 @@ func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLa
conf := conn.Conf.Clone()
conf.Type = GitlabProjectConnection
conf.Options = map[string]string{
"group": group.Name,
"group": group.FullPath,
"group-id": strconv.Itoa(group.ID),
"project": project.Name,
"project-id": strconv.Itoa(project.ID),
"url": conn.Conf.Options["url"],
}
asset := &inventory.Asset{
Name: project.NameWithNamespace,
Connections: []*inventory.Config{conf},
}

s.detectAsProject(asset, group, project)
s.detectAsProject(asset, group.ID, group.FullPath, project)
if err != nil {
return nil, nil, err
}
Expand All @@ -173,6 +184,7 @@ func (s *Service) discoverProjects(root *inventory.Asset, conn *connection.GitLa
}

func discoverGroupProjects(conn *connection.GitLabConnection, gid interface{}) ([]*gitlab.Project, error) {
log.Debug().Msgf("discover group projects for %v", gid)
perPage := 50
page := 1
total := 50
Expand All @@ -190,15 +202,17 @@ func discoverGroupProjects(conn *connection.GitLabConnection, gid interface{}) (
return projects, nil
}

func (s *Service) convertGitlabGroupsToAssetGroups(groups []*gitlab.Group, conn *connection.GitLabConnection) []*inventory.Asset {
func (s *Service) convertGitlabGroupsToAssetGroups(groups []*gitlab.Group, conn *connection.GitLabConnection, rootGroupPath string) []*inventory.Asset {
var list []*inventory.Asset
// convert to assets
for _, group := range groups {
conf := conn.Conf.Clone()
if conf.Options == nil {
conf.Options = map[string]string{}
}
conf.Options["group"] = group.Path
conf.Options["group"] = group.FullPath
conf.Options["group-id"] = strconv.Itoa(group.ID)
conf.Options["url"] = conn.Conf.Options["url"]
conf.Type = GitlabGroupConnection
asset := &inventory.Asset{
Connections: []*inventory.Config{conf},
Expand All @@ -213,64 +227,8 @@ func (s *Service) convertGitlabGroupsToAssetGroups(groups []*gitlab.Group, conn
return list
}

func discoverSubAndDescendantGroupsForGroup(conn *connection.GitLabConnection) ([]*gitlab.Group, error) {
gid, err := conn.GID()
if err != nil {
return nil, err
}
var list []*gitlab.Group
// discover subgroups
subgroups, err := groupSubgroups(conn, gid)
if err != nil {
return nil, err
}
list = append(list, subgroups...)
// discover descendant groups
descgroups, err := groupDescendantGroups(conn, gid)
if err != nil {
return nil, err
}
list = append(list, descgroups...)
return list, nil
}

func groupDescendantGroups(conn *connection.GitLabConnection, gid interface{}) ([]*gitlab.Group, error) {
perPage := 50
page := 1
total := 50
groups := []*gitlab.Group{}
for page*perPage <= total {
grps, resp, err := conn.Client().Groups.ListDescendantGroups(gid, &gitlab.ListDescendantGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}})
if err != nil {
return nil, err
}
groups = append(groups, grps...)
total = resp.TotalItems
page += 1
}

return groups, nil
}

func groupSubgroups(conn *connection.GitLabConnection, gid interface{}) ([]*gitlab.Group, error) {
perPage := 50
page := 1
total := 50
groups := []*gitlab.Group{}
for page*perPage <= total {
grps, resp, err := conn.Client().Groups.ListSubGroups(gid, &gitlab.ListSubGroupsOptions{ListOptions: gitlab.ListOptions{Page: page, PerPage: perPage}})
if err != nil {
return nil, err
}
groups = append(groups, grps...)
total = resp.TotalItems
page += 1
}

return groups, nil
}

func listAllGroups(conn *connection.GitLabConnection) ([]*gitlab.Group, error) {
log.Debug().Msg("calling list all groups")
perPage := 50
page := 1
total := 50
Expand Down
19 changes: 9 additions & 10 deletions providers/gitlab/provider/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,30 +225,29 @@ func (s *Service) detect(asset *inventory.Asset, conn *connection.GitLabConnecti
// we will discover the groups
return nil
}
group, err := conn.Group()
if err != nil {
return err
}

if conn.IsProject() {
project, err := conn.Project()
if err != nil {
return err
}
s.detectAsProject(asset, group, project)

s.detectAsProject(asset, conn.GroupID(), conn.GroupName(), project) // TODO fix 0
} else {
group, err := conn.Group()
if err != nil {
return err
}
s.detectAsGroup(asset, group)
}
return nil
}

func (s *Service) detectAsProject(asset *inventory.Asset, group *gitlab.Group, project *gitlab.Project) {
func (s *Service) detectAsProject(asset *inventory.Asset, groupID int, groupFullPath string, project *gitlab.Project) {
asset.Platform = projectPlatform
asset.Name = "GitLab Project " + project.Name
asset.PlatformIds = []string{
newGitLabProjectID(group.ID, project.ID),
newGitLabProjectIDFromPaths(group.Path, project.Path), // for backwards compatibility with v8
newGitLabProjectID(groupID, project.ID),
newGitLabProjectIDFromPaths(groupFullPath, project.Path), // for backwards compatibility with v8
}
}

Expand All @@ -257,7 +256,7 @@ func (s *Service) detectAsGroup(asset *inventory.Asset, group *gitlab.Group) err
asset.Name = "GitLab Group " + group.Name
asset.PlatformIds = []string{
newGitLabGroupID(group.ID),
newGitLabGroupIDFromPath(group.Path), // for backwards compatibility with v8
newGitLabGroupIDFromPath(group.FullPath), // for backwards compatibility with v8
}
return nil
}
Expand Down
10 changes: 5 additions & 5 deletions providers/network/resources/dns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ import (

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.mondoo.com/cnquery/llx"
"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/providers/network/connection"
"go.mondoo.com/cnquery/providers/network/resources"
"go.mondoo.com/cnquery/v9/llx"
"go.mondoo.com/cnquery/v9/providers-sdk/v1/inventory"
"go.mondoo.com/cnquery/v9/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v9/providers/network/connection"
"go.mondoo.com/cnquery/v9/providers/network/resources"
)

func TestResource_DNS(t *testing.T) {
Expand Down
Loading

0 comments on commit 7efebde

Please sign in to comment.