⚡️ verify github connection only once

When a user scans a single GH repository, this was not a problem, but for users that scan an entire organization with thousands of repositories, we were creating a connection for each asset and validating it thousands of times. This changes the behavior to verify the connection only once which will save us thousands of GH API requests. Signed-off-by: Salim Afiune Maya <[email protected]>
mondoohq · Dec 14, 2024 · 6ea1d0e · 6ea1d0e
1 parent a865b57
commit 6ea1d0e
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 9 deletions.
diff --git a/providers/github/connection/connection.go b/providers/github/connection/connection.go
@@ -73,14 +73,6 @@ func NewGithubConnection(id uint32, asset *inventory.Asset) (*GithubConnection,
 	// (default behaviour is to send fake 403 response bypassing the retry logic)
 	ctx := context.WithValue(context.Background(), github.SleepUntilPrimaryRateLimitResetWhenRateLimited, true)
 
-	// perform a quick call to verify the token's validity.
-	_, resp, err := client.Meta.Zen(ctx)
-	if err != nil {
-		if resp != nil && resp.StatusCode == 401 {
-			return nil, errors.New("invalid GitHub token provided. check the value passed with the --token flag or the GITHUB_TOKEN environment variable")
-		}
-		return nil, err
-	}
 	return &GithubConnection{
 		Connection: plugin.NewConnection(id, asset),
 		asset:      asset,
@@ -105,6 +97,20 @@ func (c *GithubConnection) Context() context.Context {
 	return c.ctx
 }
 
+func (c *GithubConnection) Verify() error {
+	// perform a quick call to verify the token's validity.
+	_, resp, err := c.client.Meta.Zen(c.ctx)
+	if err != nil {
+		if resp != nil && resp.StatusCode == 401 {
+			return errors.New(
+				"invalid GitHub token provided. check the value passed with the --token flag or the GITHUB_TOKEN environment variable",
+			)
+		}
+		return err
+	}
+	return nil
+}
+
 func newGithubAppClient(conf *inventory.Config) (*github.Client, error) {
 	appIdStr := conf.Options[OPTION_APP_ID]
 	if appIdStr == "" {

diff --git a/providers/github/provider/provider.go b/providers/github/provider/provider.go
@@ -10,25 +10,34 @@ import (
 	"strings"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"go.mondoo.com/cnquery/v11/llx"
 	"go.mondoo.com/cnquery/v11/logger"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/inventory"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/plugin"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/upstream"
+	"go.mondoo.com/cnquery/v11/providers-sdk/v1/util/memoize"
 	"go.mondoo.com/cnquery/v11/providers-sdk/v1/vault"
 	"go.mondoo.com/cnquery/v11/providers/github/connection"
 	"go.mondoo.com/cnquery/v11/providers/github/resources"
 )
 
 const ConnectionType = "github"
 
+var (
+	cacheExpirationTime = 24 * time.Hour
+	cacheCleanupTime    = 48 * time.Hour
+)
+
 type Service struct {
 	*plugin.Service
+	*memoize.Memoizer
 }
 
 func Init() *Service {
 	return &Service{
-		Service: plugin.NewService(),
+		plugin.NewService(),
+		memoize.NewMemoizer(cacheExpirationTime, cacheCleanupTime),
 	}
 }
 
@@ -163,6 +172,15 @@ func (s *Service) connect(req *plugin.ConnectReq, callback plugin.ProviderCallba
 			return nil, err
 		}
 
+		// verify the connection only once
+		_, err, _ = s.Memoize("conn_verify", func() (interface{}, error) {
+			log.Trace().Msg("verifying github connection client")
+			err := conn.Verify()
+			return nil, err
+		})
+		if err != nil {
+			return nil, err
+		}
 		var upstream *upstream.UpstreamClient
 		if req.Upstream != nil && !req.Upstream.Incognito {
 			upstream, err = req.Upstream.InitClient(context.Background())