🐛 fix github discovery for v9

providers-sdk/v1/util/convert/pointers.go

@@ -38,6 +38,13 @@ func ToInt64From32(ptr *int32) int64 {
 	return int64(*ptr)
 }
 
+func ToInt64FromInt(ptr *int) int64 {
+	if ptr == nil {
+		return 0
+	}
+	return int64(*ptr)
+}
+
 func ToFloat64(ptr *float64) float64 {
 	if ptr == nil {
 		return 0

providers/github/connection/connection.go

@@ -60,7 +60,7 @@ func NewGithubConnection(id uint32, asset *inventory.Asset, conf *inventory.Conf
 	// perform a quick call to verify the token's validity.
 	_, resp, err := client.Zen(context.Background())
 	if err != nil {
-		if resp.StatusCode == 401 {
+		if resp != nil && resp.StatusCode == 401 {
 			return nil, errors.New("invalid GitHub token provided. check the value passed with the --token flag or the GITHUB_TOKEN environment variable")
 		}
 		return nil, err

providers/github/connection/platform.go

@@ -11,6 +11,16 @@ import (
 	"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
 )
 
+const (
+	DiscoveryAll          = "all"
+	DiscoveryAuto         = "auto"
+	DiscoveryRepos        = "repos"
+	DiscoveryUsers        = "users"
+	DiscoveryRepository   = "repository" // deprecated: use repos
+	DiscoveryUser         = "user"       // deprecated: use users
+	DiscoveryOrganization = "organization"
+)
+
 var (
 	GithubRepoPlatform = &inventory.Platform{
 		Name:    "github-repo",

providers/github/provider/provider.go

@@ -38,9 +38,14 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error)
 		flags = map[string]*llx.Primitive{}
 	}
 
+	if len(req.Args) == 0 {
+		return nil, errors.New("invalid. must specify org, repo, or user")
+	}
+
 	conf := &inventory.Config{
-		Type:    req.Connector,
-		Options: map[string]string{},
+		Type:     req.Connector,
+		Options:  map[string]string{},
+		Discover: &inventory.Discovery{},
 	}
 
 	token := ""
@@ -55,6 +60,18 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error)
 	}
 	conf.Credentials = append(conf.Credentials, vault.NewPasswordCredential("", token))
 
+	// discovery flags
+	discoverTargets := []string{}
+	if x, ok := flags["discover"]; ok && len(x.Array) != 0 {
+		for i := range x.Array {
+			entry := string(x.Array[i].Value)
+			discoverTargets = append(discoverTargets, entry)
+		}
+	} else {
+		discoverTargets = []string{"auto"}
+	}
+	conf.Discover = &inventory.Discovery{Targets: discoverTargets}
+
 	// Do custom flag parsing here
 	switch req.Args[0] {
 	case "org":
@@ -64,7 +81,7 @@ func (s *Service) ParseCLI(req *plugin.ParseCLIReq) (*plugin.ParseCLIRes, error)
 	case "repo":
 		conf.Options["repository"] = req.Args[1]
 	default:
-		return nil, errors.New("invalid GitHub sub-command")
+		return nil, errors.New("invalid GitHub sub-command, supported are: org, user, or repo")
 	}
 
 	asset := inventory.Asset{
@@ -90,12 +107,16 @@ func (s *Service) Connect(req *plugin.ConnectReq, callback plugin.ProviderCallba
 			return nil, err
 		}
 	}
+	inv, err := s.discover(conn)
+	if err != nil {
+		return nil, err
+	}
 
 	return &plugin.ConnectRes{
 		Id:        conn.ID(),
 		Name:      conn.Name(),
 		Asset:     req.Asset,
-		Inventory: nil,
+		Inventory: inv,
 	}, nil
 }
 
@@ -168,7 +189,6 @@ func (s *Service) detect(asset *inventory.Asset, conn *connection.GithubConnecti
 	if err != nil {
 		return err
 	}
-	// TODO: Add platform IDs
 	asset.PlatformIds = []string{id}
 	return nil
 }
@@ -259,3 +279,17 @@ func (s *Service) StoreData(req *plugin.StoreReq) (*plugin.StoreRes, error) {
 	}
 	return &plugin.StoreRes{}, nil
 }
+
+func (s *Service) discover(conn *connection.GithubConnection) (*inventory.Inventory, error) {
+	if conn.Conf.Discover == nil {
+		return nil, nil
+	}
+
+	runtime, ok := s.runtimes[conn.ID()]
+	if !ok {
+		// no connection found, this should never happen
+		return nil, errors.New("connection " + strconv.FormatUint(uint64(conn.ID()), 10) + " not found")
+	}
+
+	return resources.Discover(runtime, conn.Conf.Options)
+}

providers/github/resources/discovery.go

@@ -0,0 +1,198 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package resources
+
+import (
+	"go.mondoo.com/cnquery/llx"
+	"go.mondoo.com/cnquery/providers-sdk/v1/inventory"
+	"go.mondoo.com/cnquery/providers-sdk/v1/plugin"
+	"go.mondoo.com/cnquery/providers/github/connection"
+	"go.mondoo.com/cnquery/utils/stringx"
+)
+
+func Discover(runtime *plugin.Runtime, opts map[string]string) (*inventory.Inventory, error) {
+	conn := runtime.Connection.(*connection.GithubConnection)
+
+	in := &inventory.Inventory{Spec: &inventory.InventorySpec{
+		Assets: []*inventory.Asset{},
+	}}
+
+	targets := handleTargets(conn.Conf.Discover.Targets)
+	list, err := discover(runtime, targets)
+	if err != nil {
+		return in, err
+	}
+
+	in.Spec.Assets = list
+	return in, nil
+}
+
+func handleTargets(targets []string) []string {
+	if stringx.Contains(targets, connection.DiscoveryAll) {
+		return []string{connection.DiscoveryRepos, connection.DiscoveryUsers}
+	}
+	return targets
+}
+
+func discover(runtime *plugin.Runtime, targets []string) ([]*inventory.Asset, error) {
+	conn := runtime.Connection.(*connection.GithubConnection)
+	assetList := []*inventory.Asset{}
+	if orgName := conn.Conf.Options["organization"]; orgName != "" {
+		orgAssets, err := org(runtime, orgName, conn, targets)
+		if err != nil {
+			return nil, err
+		}
+		assetList = append(assetList, orgAssets...)
+	}
+
+	repoName := conn.Conf.Options["repository"]
+	var owner string
+	repoId := conn.Conf.Options["repository"]
+	if repoId != "" {
+		owner = conn.Conf.Options["owner"]
+		if owner == "" {
+			owner = conn.Conf.Options["organization"]
+		}
+		if owner == "" {
+			owner = conn.Conf.Options["user"]
+		}
+	}
+	if repoName != "" && owner != "" {
+		repoAssets, err := repo(runtime, repoName, owner, conn, targets)
+		if err != nil {
+			return nil, err
+		}
+		assetList = append(assetList, repoAssets...)
+	}
+
+	userId := conn.Conf.Options["user"]
+	if userId == "" {
+		userId = conn.Conf.Options["owner"]
+	}
+	if conn.Conf.Options["user"] != "" {
+		userAssets, err := user(runtime, userId, conn)
+		if err != nil {
+			return nil, err
+		}
+		assetList = append(assetList, userAssets...)
+	}
+
+	return assetList, nil
+}
+
+func cloneInventoryConf(invConf *inventory.Config) *inventory.Config {
+	invConfClone := invConf.Clone()
+	// We do not want to run discovery again for the already discovered assets
+	invConfClone.Discover = &inventory.Discovery{}
+	return invConfClone
+}
+
+func org(runtime *plugin.Runtime, orgName string, conn *connection.GithubConnection, targets []string) ([]*inventory.Asset, error) {
+	assetList := []*inventory.Asset{}
+	org, err := getMqlGithubOrg(runtime, orgName)
+	if err != nil {
+		return nil, err
+	}
+	assetList = append(assetList, &inventory.Asset{
+		PlatformIds: []string{connection.NewGithubOrgIdentifier(org.Name.Data)},
+		Name:        org.Name.Data,
+		Platform:    connection.GithubOrgPlatform,
+		Labels:      map[string]string{},
+		Connections: []*inventory.Config{cloneInventoryConf(conn.Conf)},
+	})
+	if stringx.Contains(targets, connection.DiscoveryRepos) || stringx.Contains(targets, connection.DiscoveryRepository) || stringx.Contains(targets, connection.DiscoveryAll) || stringx.Contains(targets, connection.DiscoveryAuto) {
+		if stringx.Contains(targets, connection.DiscoveryRepos) || stringx.Contains(targets, connection.DiscoveryRepository) {
+			assetList = []*inventory.Asset{}
+		}
+		for i := range org.GetRepositories().Data {
+			repo := org.GetRepositories().Data[i].(*mqlGithubRepository)
+			assetList = append(assetList, &inventory.Asset{
+				PlatformIds: []string{connection.NewGitHubRepoIdentifier(org.Name.Data, repo.Name.Data)},
+				Name:        org.Name.Data + "/" + repo.Name.Data,
+				Platform:    connection.GithubRepoPlatform,
+				Labels:      make(map[string]string),
+				Connections: []*inventory.Config{cloneInventoryConf(conn.Conf)},
+			})
+		}
+	}
+	if stringx.Contains(targets, connection.DiscoveryUsers) || stringx.Contains(targets, connection.DiscoveryUser) {
+		assetList = []*inventory.Asset{}
+		for i := range org.GetMembers().Data {
+			user := org.GetMembers().Data[i].(*mqlGithubUser)
+			if user.Name.Data == "" {
+				continue
+			}
+			assetList = append(assetList, &inventory.Asset{
+				PlatformIds: []string{connection.NewGithubUserIdentifier(user.Login.Data)},
+				Name:        user.Name.Data,
+				Platform:    connection.GithubUserPlatform,
+				Labels:      make(map[string]string),
+				Connections: []*inventory.Config{cloneInventoryConf(conn.Conf)},
+			})
+		}
+	}
+	return assetList, nil
+}
+
+func getMqlGithubOrg(runtime *plugin.Runtime, orgName string) (*mqlGithubOrganization, error) {
+	res, err := NewResource(runtime, "github.organization", map[string]*llx.RawData{"name": llx.StringData(orgName)})
+	if err != nil {
+		return nil, err
+	}
+	return res.(*mqlGithubOrganization), nil
+}
+
+func repo(runtime *plugin.Runtime, repoName string, owner string, conn *connection.GithubConnection, targets []string) ([]*inventory.Asset, error) {
+	assetList := []*inventory.Asset{}
+
+	repo, err := getMqlGithubRepo(runtime, repoName)
+	if err != nil {
+		return nil, err
+	}
+
+	assetList = append(assetList, &inventory.Asset{
+		PlatformIds: []string{connection.NewGitHubRepoIdentifier(owner, repo.Name.Data)},
+		Name:        owner + "/" + repo.Name.Data,
+		Platform:    connection.GithubRepoPlatform,
+		Labels:      make(map[string]string),
+		Connections: []*inventory.Config{cloneInventoryConf(conn.Conf)},
+	})
+
+	return assetList, nil
+}
+
+func getMqlGithubRepo(runtime *plugin.Runtime, repoName string) (*mqlGithubRepository, error) {
+	res, err := NewResource(runtime, "github.repository", map[string]*llx.RawData{"name": llx.StringData(repoName)})
+	if err != nil {
+		return nil, err
+	}
+
+	return res.(*mqlGithubRepository), nil
+}
+
+func user(runtime *plugin.Runtime, userName string, conn *connection.GithubConnection) ([]*inventory.Asset, error) {
+	assetList := []*inventory.Asset{}
+
+	user, err := getMqlGithubUser(runtime, userName)
+	if err != nil {
+		return nil, err
+	}
+	assetList = append(assetList, &inventory.Asset{
+		PlatformIds: []string{connection.NewGithubUserIdentifier(user.Login.Data)},
+		Name:        user.Name.Data,
+		Platform:    connection.GithubUserPlatform,
+		Labels:      make(map[string]string),
+		Connections: []*inventory.Config{cloneInventoryConf(conn.Conf)},
+	})
+	return assetList, nil
+}
+
+func getMqlGithubUser(runtime *plugin.Runtime, userName string) (*mqlGithubUser, error) {
+	res, err := NewResource(runtime, "github.user", map[string]*llx.RawData{"name": llx.StringData(userName)})
+	if err != nil {
+		return nil, err
+	}
+
+	return res.(*mqlGithubUser), nil
+}