diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..082b1943 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "makefile.configureOnOpen": false +} \ No newline at end of file diff --git a/command/cmd/describer.go b/command/cmd/describer.go index 8068ff45..c0252d7f 100644 --- a/command/cmd/describer.go +++ b/command/cmd/describer.go @@ -3,6 +3,11 @@ package cmd import ( "encoding/json" "fmt" + "os" + "strconv" + "strings" + "time" + "github.com/google/uuid" "github.com/opengovern/og-describer-github/pkg/describer" model "github.com/opengovern/og-describer-github/pkg/sdk/models" @@ -14,10 +19,6 @@ import ( "github.com/spf13/cobra" "go.uber.org/zap" "golang.org/x/net/context" - "os" - "strconv" - "strings" - "time" ) var ( @@ -46,7 +47,7 @@ var describerCmd = &cobra.Command{ IntegrationType: configs.IntegrationTypeLower, CipherText: "", IntegrationLabels: map[string]string{ - "OrganizationName": OrganizationName, + "OrganizationName": "opengovern", }, IntegrationAnnotations: nil, } @@ -55,7 +56,7 @@ var describerCmd = &cobra.Command{ logger, _ := zap.NewProduction() creds, err := provider.AccountCredentialsFromMap(map[string]any{ - "pat_token": PatToken, + "pat_token": "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", }) if err != nil { return fmt.Errorf(" account credentials: %w", err) diff --git a/provider/describer/action_repository_workflow_run.go b/provider/describer/action_repository_workflow_run.go index b51b83e7..904c105c 100644 --- a/provider/describer/action_repository_workflow_run.go +++ b/provider/describer/action_repository_workflow_run.go @@ -4,44 +4,29 @@ import ( "context" "encoding/json" "fmt" - "github.com/opengovern/og-describer-github/pkg/sdk/models" - "github.com/opengovern/og-describer-github/provider/model" - resilientbridge "github.com/opengovern/resilient-bridge" - "github.com/opengovern/resilient-bridge/adapters" "log" "net/url" "strconv" "strings" + + "github.com/opengovern/og-describer-github/pkg/sdk/models" + "github.com/opengovern/og-describer-github/provider/model" + resilientbridge "github.com/opengovern/resilient-bridge" ) func GetAllWorkflowRuns(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender) ([]models.Resource, error) { - client := githubClient.RestClient - owner := organizationName - repositories, err := getRepositories(ctx, client, owner) + // Retrieve only active (non-archived, non-disabled) repositories + repositories, err := GetRepositoryListWithOptions(ctx, githubClient, organizationName, nil, true, true) if err != nil { - return nil, nil + return nil, fmt.Errorf("error fetching repositories for workflow runs: %w", err) } - sdk := resilientbridge.NewResilientBridge() - sdk.RegisterProvider("github", adapters.NewGitHubAdapter(githubClient.Token), &resilientbridge.ProviderConfig{ - UseProviderLimits: true, - MaxRetries: 3, - BaseBackoff: 0, - }) + sdk := newResilientSDK(githubClient.Token) var values []models.Resource for _, repo := range repositories { - active, err := checkRepositoryActive(sdk, owner, repo.GetName()) - if err != nil { - log.Fatalf("Error checking repository: %v", err) - } - - if !active { - // Repository is archived or disabled, return 0 workflow runs - // No output needed, just exit gracefully. - continue - } - repoValues, err := GetRepositoryWorkflowRuns(ctx, sdk, stream, owner, repo.GetName()) + // repo.Name should be the repository name field from the returned resources + repoValues, err := GetRepositoryWorkflowRuns(ctx, sdk, stream, organizationName, repo.Name) if err != nil { return nil, err } diff --git a/provider/describer/commit.go b/provider/describer/commit.go index 37f8bf92..fb83497b 100644 --- a/provider/describer/commit.go +++ b/provider/describer/commit.go @@ -1,42 +1,37 @@ +// commit.go package describer import ( "context" "encoding/json" "fmt" + "log" + "os" + "strconv" + "sync" + "github.com/opengovern/og-describer-github/pkg/sdk/models" "github.com/opengovern/og-describer-github/provider/model" resilientbridge "github.com/opengovern/resilient-bridge" "github.com/opengovern/resilient-bridge/adapters" - "log" ) +// ListCommits fetches commits from all active repositories under the specified organization. +// If a stream is provided, each commit is sent to the stream as it’s processed. +// Otherwise, commits are collected and returned as a slice. func ListCommits(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender) ([]models.Resource, error) { - repositories, err := getRepositories(ctx, githubClient.RestClient, organizationName) + // Retrieve repositories while excluding archived and disabled ones + repos, err := GetRepositoryListWithOptions(ctx, githubClient, organizationName, nil, true, true) if err != nil { return nil, err } - sdk := resilientbridge.NewResilientBridge() - sdk.RegisterProvider("github", adapters.NewGitHubAdapter(githubClient.Token), &resilientbridge.ProviderConfig{ - UseProviderLimits: true, - MaxRetries: 3, - BaseBackoff: 0, - }) + sdk := newResilientSDK(githubClient.Token) var values []models.Resource - for _, repo := range repositories { - active, err := checkRepositoryActive(sdk, organizationName, repo.GetName()) - if err != nil { - return nil, err - } - - if !active { - // Repository is archived or disabled, return 0 commits - // No output needed, just exit gracefully. - continue - } - repoValues, err := GetRepositoryCommits(ctx, sdk, stream, organizationName, repo.GetName()) + for _, r := range repos { + // r.Name should correspond to the repository name + repoValues, err := GetRepositoryCommits(ctx, sdk, stream, organizationName, r.Name) if err != nil { return nil, err } @@ -46,211 +41,95 @@ func ListCommits(ctx context.Context, githubClient GitHubClient, organizationNam return values, nil } +// GetRepositoryCommits fetches up to 50 commits for a single repository. +// If a stream is provided, commits are streamed; otherwise, returns them as a slice. func GetRepositoryCommits(ctx context.Context, sdk *resilientbridge.ResilientBridge, stream *models.StreamSender, owner, repo string) ([]models.Resource, error) { - maxCommits := 50 + //const maxCommits = 50 + const maxCommits = 25 commits, err := fetchCommitList(sdk, owner, repo, maxCommits) if err != nil { - log.Fatalf("Error fetching commits list: %v", err) + return nil, fmt.Errorf("error fetching commits list for %s/%s: %w", owner, repo, err) } - var values []models.Resource - for _, c := range commits { - commitJSON, err := fetchCommitDetails(sdk, owner, repo, c.SHA) - if err != nil { - log.Printf("Error fetching commit %s details: %v", c.SHA, err) - continue + // Determine concurrency level from env or default to 5 + concurrency := 3 + if cStr := os.Getenv("CONCURRENCY"); cStr != "" { + if cVal, err := strconv.Atoi(cStr); err == nil && cVal > 0 { + concurrency = cVal } + } + log.Printf("Fetching commit details with concurrency=%d", concurrency) - var commit model.CommitDescription - - err = json.Unmarshal(commitJSON, &commit) - if err != nil { - log.Println("Error unmarshaling JSON:", err) - continue - } + results := make([]models.Resource, len(commits)) - value := models.Resource{ - ID: commit.ID, - Name: commit.ID, - Description: JSONAllFieldsMarshaller{ - Value: commit, - }, - } - if stream != nil { - if err := (*stream)(value); err != nil { - return nil, err - } - } else { - values = append(values, value) - } + type job struct { + index int + sha string } + jobCh := make(chan job) + wg := sync.WaitGroup{} - return values, nil -} - -//func GetAllCommits(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender) ([]models.Resource, error) { -// client := githubClient.RestClient -// owner := organizationName -// repositories, err := getRepositories(ctx, client, owner) -// if err != nil { -// return nil, nil -// } -// var values []models.Resource -// for _, repo := range repositories { -// repoValues, err := GetRepositoryCommits(ctx, githubClient, stream, owner, repo.GetName()) -// if err != nil { -// return nil, err -// } -// values = append(values, repoValues...) -// } -// return values, nil -//} -// -//func GetRepositoryCommits(ctx context.Context, githubClient GitHubClient, stream *models.StreamSender, owner, repo string) ([]models.Resource, error) { -// client := githubClient.GraphQLClient -// var query struct { -// RateLimit steampipemodels.RateLimit -// Repository struct { -// DefaultBranchRef struct { -// Target struct { -// Commit struct { -// History struct { -// TotalCount int -// PageInfo steampipemodels.PageInfo -// Nodes []steampipemodels.Commit -// } `graphql:"history(first: $pageSize, after: $cursor, since: $since, until: $until)"` -// } `graphql:"... on Commit"` -// } -// } -// } `graphql:"repository(owner: $owner, name: $name)"` -// } -// variables := map[string]interface{}{ -// "owner": githubv4.String(owner), -// "name": githubv4.String(repo), -// "pageSize": githubv4.Int(pageSize), -// "cursor": (*githubv4.String)(nil), -// "since": (*githubv4.GitTimestamp)(nil), -// "until": (*githubv4.GitTimestamp)(nil), -// } -// appendCommitColumnIncludes(&variables, commitCols()) -// repoFullName := formRepositoryFullName(owner, repo) -// var values []models.Resource -// for { -// err := client.Query(ctx, &query, variables) -// if err != nil { -// return nil, err -// } -// for _, commit := range query.Repository.DefaultBranchRef.Target.Commit.History.Nodes { -// value := models.Resource{ -// ID: commit.Sha, -// Name: commit.Sha, -// Description: JSONAllFieldsMarshaller{ -// Value: model.CommitDescription{ -// Commit: commit, -// RepoFullName: repoFullName, -// AuthorLogin: commit.Author.User.Login, -// CommitterLogin: commit.Committer.User.Login, -// }, -// }, -// } -// if stream != nil { -// if err := (*stream)(value); err != nil { -// return nil, err -// } -// } else { -// values = append(values, value) -// } -// } -// if !query.Repository.DefaultBranchRef.Target.Commit.History.PageInfo.HasNextPage { -// break -// } -// variables["cursor"] = githubv4.NewString(query.Repository.DefaultBranchRef.Target.Commit.History.PageInfo.EndCursor) -// } -// return values, nil -//} -// -//func GetRepositoryCommit(ctx context.Context, githubClient GitHubClient, organizationName string, repositoryName string, resourceID string, stream *models.StreamSender) (*models.Resource, error) { -// repoFullName := formRepositoryFullName(organizationName, repositoryName) -// -// var query struct { -// RateLimit steampipemodels.RateLimit -// Repository struct { -// Object struct { -// Commit steampipemodels.Commit `graphql:"... on Commit"` -// } `graphql:"object(oid: $sha)"` -// } `graphql:"repository(owner: $owner, name: $name)"` -// } -// -// variables := map[string]interface{}{ -// "owner": githubv4.String(organizationName), -// "name": githubv4.String(repositoryName), -// "sha": githubv4.GitObjectID(resourceID), -// } -// -// client := githubClient.GraphQLClient -// appendCommitColumnIncludes(&variables, commitCols()) -// -// err := client.Query(ctx, &query, variables) -// if err != nil { -// return nil, err -// } -// -// value := models.Resource{ -// ID: query.Repository.Object.Commit.Sha, -// Name: query.Repository.Object.Commit.Sha, -// Description: JSONAllFieldsMarshaller{ -// Value: model.CommitDescription{ -// Commit: query.Repository.Object.Commit, -// RepoFullName: repoFullName, -// AuthorLogin: query.Repository.Object.Commit.Author.User.Login, -// CommitterLogin: query.Repository.Object.Commit.Committer.User.Login, -// }, -// }, -// } -// if stream != nil { -// if err := (*stream)(value); err != nil { -// return nil, err -// } -// } -// -// return &value, nil -//} + worker := func() { + defer wg.Done() + for j := range jobCh { + commitJSON, err := fetchCommitDetails(sdk, owner, repo, j.sha) + if err != nil { + log.Printf("Error fetching commit %s details: %v", j.sha, err) + continue + } -type commitRef struct { - SHA string `json:"sha"` -} + var commit model.CommitDescription + if err := json.Unmarshal(commitJSON, &commit); err != nil { + log.Printf("Error unmarshaling JSON for commit %s: %v", j.sha, err) + continue + } -// checkRepositoryActive returns false if the repository is archived or disabled, true otherwise. -func checkRepositoryActive(sdk *resilientbridge.ResilientBridge, owner, repo string) (bool, error) { - req := &resilientbridge.NormalizedRequest{ - Method: "GET", - Endpoint: fmt.Sprintf("/repos/%s/%s", owner, repo), - Headers: map[string]string{"Accept": "application/vnd.github+json"}, + value := models.Resource{ + ID: commit.ID, + Name: commit.ID, + Description: JSONAllFieldsMarshaller{ + Value: commit, + }, + } + results[j.index] = value + } } - resp, err := sdk.Request("github", req) - if err != nil { - return false, fmt.Errorf("error checking repository: %w", err) - } - if resp.StatusCode == 404 { - // Repo not found, treat as inactive - return false, nil - } else if resp.StatusCode >= 400 { - return false, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) + for i := 0; i < concurrency; i++ { + wg.Add(1) + go worker() } - var repoInfo struct { - Archived bool `json:"archived"` - Disabled bool `json:"disabled"` + for i, c := range commits { + jobCh <- job{index: i, sha: c.SHA} } - if err := json.Unmarshal(resp.Data, &repoInfo); err != nil { - return false, fmt.Errorf("error decoding repository info: %w", err) + close(jobCh) + + wg.Wait() + + if stream != nil { + for _, res := range results { + if res.ID == "" { + continue + } + if err := (*stream)(res); err != nil { + return nil, err + } + } + return nil, nil } - if repoInfo.Archived || repoInfo.Disabled { - return false, nil + var finalResults []models.Resource + for _, res := range results { + if res.ID != "" { + finalResults = append(finalResults, res) + } } - return true, nil + return finalResults, nil +} + +type commitRef struct { + SHA string `json:"sha"` } // fetchCommitList returns up to maxCommits commit references from the repo’s default branch. @@ -277,6 +156,12 @@ func fetchCommitList(sdk *resilientbridge.ResilientBridge, owner, repo string, m return nil, fmt.Errorf("error fetching commits: %w", err) } + // Handle HTTP errors + if resp.StatusCode == 409 { + // 409 typically means no commits on default branch or empty repo + // Treat this as no commits found. + return []commitRef{}, nil + } if resp.StatusCode >= 400 { return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) } @@ -306,7 +191,6 @@ func fetchCommitList(sdk *resilientbridge.ResilientBridge, owner, repo string, m } func fetchCommitDetails(sdk *resilientbridge.ResilientBridge, owner, repo, sha string) ([]byte, error) { - // Fetch the commit details req := &resilientbridge.NormalizedRequest{ Method: "GET", Endpoint: fmt.Sprintf("/repos/%s/%s/commits/%s", owner, repo, sha), @@ -320,428 +204,15 @@ func fetchCommitDetails(sdk *resilientbridge.ResilientBridge, owner, repo, sha s return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) } - var commitData map[string]interface{} - if err := json.Unmarshal(resp.Data, &commitData); err != nil { - return nil, fmt.Errorf("error unmarshaling commit details: %w", err) - } - - // Helper functions - getString := func(m map[string]interface{}, key string) *string { - if m == nil { - return nil - } - if val, ok := m[key].(string); ok { - return &val - } - return nil - } - getFloat := func(m map[string]interface{}, key string) *int { - if m == nil { - return nil - } - if val, ok := m[key].(float64); ok { - v := int(val) - return &v - } - return nil - } - getBool := func(m map[string]interface{}, key string) *bool { - if m == nil { - return nil - } - if val, ok := m[key].(bool); ok { - return &val - } - return nil - } - - commitSha := getString(commitData, "sha") - htmlURL := getString(commitData, "html_url") - nodeID := getString(commitData, "node_id") - - commitSection, _ := commitData["commit"].(map[string]interface{}) - message := getString(commitSection, "message") - - var date *string - var commitAuthor map[string]interface{} - if commitSection != nil { - if ca, ok := commitSection["author"].(map[string]interface{}); ok { - commitAuthor = ca - date = getString(ca, "date") - } - } - - // stats - var stats map[string]interface{} - if s, ok := commitData["stats"].(map[string]interface{}); ok { - stats = s - } - additions := getFloat(stats, "additions") - deletions := getFloat(stats, "deletions") - total := getFloat(stats, "total") - - // author - authorObj := map[string]interface{}{ - "email": nil, - "html_url": nil, - "id": nil, - "login": nil, - "name": nil, - "node_id": nil, - "type": nil, - } - if commitAuthor != nil { - if email := getString(commitAuthor, "email"); email != nil { - authorObj["email"] = *email - } - if name := getString(commitAuthor, "name"); name != nil { - authorObj["name"] = *name - } - } - - if topAuthor, ok := commitData["author"].(map[string]interface{}); ok { - if login := getString(topAuthor, "login"); login != nil { - authorObj["login"] = *login - } - if idVal, ok := topAuthor["id"].(float64); ok { - authorObj["id"] = int(idVal) - } - if n := getString(topAuthor, "node_id"); n != nil { - authorObj["node_id"] = *n - } - if h := getString(topAuthor, "html_url"); h != nil { - authorObj["html_url"] = *h - } - if t := getString(topAuthor, "type"); t != nil { - authorObj["type"] = *t - } - } - - // files - filesArray := []interface{}{} - if files, ok := commitData["files"].([]interface{}); ok { - for _, f := range files { - if fm, ok := f.(map[string]interface{}); ok { - newFile := map[string]interface{}{ - "additions": nil, - "changes": nil, - "deletions": nil, - "filename": nil, - "sha": nil, - "status": nil, - } - if a := getFloat(fm, "additions"); a != nil { - newFile["additions"] = *a - } - if c := getFloat(fm, "changes"); c != nil { - newFile["changes"] = *c - } - if d := getFloat(fm, "deletions"); d != nil { - newFile["deletions"] = *d - } - if fn := getString(fm, "filename"); fn != nil { - newFile["filename"] = *fn - } - if sh := getString(fm, "sha"); sh != nil { - newFile["sha"] = *sh - } - if st := getString(fm, "status"); st != nil { - newFile["status"] = *st - } - filesArray = append(filesArray, newFile) - } - } - } - - // parents - parentsArray := []interface{}{} - if parents, ok := commitData["parents"].([]interface{}); ok { - for _, p := range parents { - if pm, ok := p.(map[string]interface{}); ok { - newParent := map[string]interface{}{ - "sha": nil, - } - if ps := getString(pm, "sha"); ps != nil { - newParent["sha"] = *ps - } - parentsArray = append(parentsArray, newParent) - } - } - } - - // comment_count - var commentCount *int - if commitSection != nil { - commentCount = getFloat(commitSection, "comment_count") - } - - // tree (only sha) - var treeObj map[string]interface{} - if commitSection != nil { - if tree, ok := commitSection["tree"].(map[string]interface{}); ok { - treeObj = map[string]interface{}{ - "sha": nil, - } - if tsha := getString(tree, "sha"); tsha != nil { - treeObj["sha"] = *tsha - } - } - } - - // verification details - var isVerified *bool - var verificationDetails map[string]interface{} - if commitSection != nil { - if verification, ok := commitSection["verification"].(map[string]interface{}); ok { - isVerified = getBool(verification, "verified") - reason := getString(verification, "reason") - signature := getString(verification, "signature") - verifiedAt := getString(verification, "verified_at") - - verificationDetails = map[string]interface{}{ - "reason": nil, - "signature": nil, - "verified_at": nil, - } - if reason != nil { - verificationDetails["reason"] = *reason - } - if signature != nil { - verificationDetails["signature"] = *signature - } - if verifiedAt != nil { - verificationDetails["verified_at"] = *verifiedAt - } - } - } - - // additional_details - additionalDetailsObj := map[string]interface{}{ - "node_id": nil, - "parents": parentsArray, - "tree": nil, - "verification_details": nil, - } - if nodeID != nil { - additionalDetailsObj["node_id"] = *nodeID - } - if treeObj != nil { - additionalDetailsObj["tree"] = treeObj - } - if verificationDetails != nil { - additionalDetailsObj["verification_details"] = verificationDetails - } - - // Fetch associated pull requests - prs, err := fetchPullRequestsForCommit(sdk, owner, repo, sha) - if err != nil { - prs = []int{} - } - - // Determine the branch: - // If we have at least one PR, fetch the first PR details and use its base.ref as the branch. - var branchName string - if len(prs) > 0 { - prBranch, err := fetchFirstPRBranch(sdk, owner, repo, prs[0]) - if err == nil && prBranch != "" { - branchName = prBranch - } - } - - // If no PR-based branch found, fallback to findBranchByCommit - if branchName == "" { - bname, berr := findBranchByCommit(sdk, owner, repo, sha) - if berr == nil && bname != "" { - branchName = bname - } - } - - // target - targetObj := map[string]interface{}{ - "branch": nil, - "organization": owner, - "repository": repo, - } - if branchName != "" { - targetObj["branch"] = branchName - } - - // Convert pointers to interface{} - finalID := func() interface{} { - if commitSha == nil { - return nil - } - return *commitSha - }() - finalDate := func() interface{} { - if date == nil { - return nil - } - return *date - }() - finalMessage := func() interface{} { - if message == nil { - return nil - } - return *message - }() - finalHtmlURL := func() interface{} { - if htmlURL == nil { - return nil - } - return *htmlURL - }() - finalIsVerified := func() interface{} { - if isVerified == nil { - return nil - } - return *isVerified - }() - finalCommentCount := func() interface{} { - if commentCount == nil { - return nil - } - return *commentCount - }() - finalAdditions := func() interface{} { - if additions == nil { - return nil - } - return *additions - }() - finalDeletions := func() interface{} { - if deletions == nil { - return nil - } - return *deletions - }() - finalTotal := func() interface{} { - if total == nil { - return nil - } - return *total - }() - - output := map[string]interface{}{ - "id": finalID, - "date": finalDate, - "message": finalMessage, - "html_url": finalHtmlURL, - "target": targetObj, - "is_verified": finalIsVerified, - "author": authorObj, - "changes": map[string]interface{}{ - "additions": finalAdditions, - "deletions": finalDeletions, - "total": finalTotal, - }, - "comment_count": finalCommentCount, - "additional_details": additionalDetailsObj, - "files": filesArray, - "pull_requests": prs, - } - - modifiedData, err := json.MarshalIndent(output, "", " ") - if err != nil { - return nil, fmt.Errorf("error encoding final commit details: %w", err) - } - - return modifiedData, nil -} - -func fetchPullRequestsForCommit(sdk *resilientbridge.ResilientBridge, owner, repo, sha string) ([]int, error) { - req := &resilientbridge.NormalizedRequest{ - Method: "GET", - Endpoint: fmt.Sprintf("/repos/%s/%s/commits/%s/pulls", owner, repo, sha), - Headers: map[string]string{"Accept": "application/vnd.github+json"}, - } - - resp, err := sdk.Request("github", req) - if err != nil { - return nil, fmt.Errorf("error fetching pull requests for commit: %w", err) - } - - if resp.StatusCode == 409 || resp.StatusCode == 404 { - return []int{}, nil - } - if resp.StatusCode >= 400 { - return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) - } - - var pulls []map[string]interface{} - if err := json.Unmarshal(resp.Data, &pulls); err != nil { - return nil, fmt.Errorf("error decoding pull requests: %w", err) - } - - var prNumbers []int - for _, pr := range pulls { - if num, ok := pr["number"].(float64); ok { - prNumbers = append(prNumbers, int(num)) - } - } - - return prNumbers, nil -} - -// fetchFirstPRBranch fetches details of a given pull request number and returns the base branch -func fetchFirstPRBranch(sdk *resilientbridge.ResilientBridge, owner, repo string, prNumber int) (string, error) { - req := &resilientbridge.NormalizedRequest{ - Method: "GET", - Endpoint: fmt.Sprintf("/repos/%s/%s/pulls/%d", owner, repo, prNumber), - Headers: map[string]string{"Accept": "application/vnd.github+json"}, - } - resp, err := sdk.Request("github", req) - if err != nil { - return "", fmt.Errorf("error fetching pull request details: %w", err) - } - if resp.StatusCode >= 400 { - return "", fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) - } - - var prData map[string]interface{} - if err := json.Unmarshal(resp.Data, &prData); err != nil { - return "", fmt.Errorf("error decoding pull request details: %w", err) - } - - base, ok := prData["base"].(map[string]interface{}) - if !ok { - return "", nil - } - - ref, ok := base["ref"].(string) - if !ok { - return "", nil - } - - return ref, nil + return resp.Data, nil } -func findBranchByCommit(sdk *resilientbridge.ResilientBridge, owner, repo, sha string) (string, error) { - req := &resilientbridge.NormalizedRequest{ - Method: "GET", - Endpoint: fmt.Sprintf("/repos/%s/%s/branches?sha=%s", owner, repo, sha), - Headers: map[string]string{"Accept": "application/vnd.github+json"}, - } - resp, err := sdk.Request("github", req) - if err != nil { - return "", fmt.Errorf("error fetching branches for commit: %w", err) - } - if resp.StatusCode >= 400 { - return "", fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) - } - - var branches []map[string]interface{} - if err := json.Unmarshal(resp.Data, &branches); err != nil { - return "", fmt.Errorf("error decoding branches: %w", err) - } - - if len(branches) == 0 { - return "", nil - } - - if name, ok := branches[0]["name"].(string); ok && name != "" { - return name, nil - } - - return "", nil +func newResilientSDK(token string) *resilientbridge.ResilientBridge { + sdk := resilientbridge.NewResilientBridge() + sdk.RegisterProvider("github", adapters.NewGitHubAdapter(token), &resilientbridge.ProviderConfig{ + UseProviderLimits: true, + MaxRetries: 3, + BaseBackoff: 0, + }) + return sdk } diff --git a/provider/describer/repository.go b/provider/describer/repository.go index 92d66f7b..3c46e4b0 100644 --- a/provider/describer/repository.go +++ b/provider/describer/repository.go @@ -4,18 +4,27 @@ import ( "context" "encoding/json" "fmt" - "github.com/google/go-github/v55/github" - "github.com/opengovern/og-describer-github/pkg/sdk/models" - "github.com/opengovern/og-describer-github/provider/model" - resilientbridge "github.com/opengovern/resilient-bridge" - "github.com/opengovern/resilient-bridge/adapters" "log" "regexp" "strconv" "strings" + "sync" + + "github.com/opengovern/og-describer-github/pkg/sdk/models" + "github.com/opengovern/og-describer-github/provider/model" + resilientbridge "github.com/opengovern/resilient-bridge" + "github.com/opengovern/resilient-bridge/adapters" ) +// GetRepositoryList returns a list of all active (non-archived, non-disabled) repos in the organization. +// By default, no excludes are applied, so this returns only active repositories. func GetRepositoryList(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender) ([]models.Resource, error) { + // Call the helper with default options (no excludes) + return GetRepositoryListWithOptions(ctx, githubClient, organizationName, stream, false, false) +} + +// GetRepositoryListWithOptions returns a list of all active repos in the organization with options to exclude archived or disabled. +func GetRepositoryListWithOptions(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender, excludeArchived bool, excludeDisabled bool) ([]models.Resource, error) { maxResults := 100 sdk := resilientbridge.NewResilientBridge() @@ -25,379 +34,135 @@ func GetRepositoryList(ctx context.Context, githubClient GitHubClient, organizat BaseBackoff: 0, }) - allRepos, err := fetchOrgRepos(sdk, organizationName, maxResults) + allRepos, err := _fetchOrgRepos(sdk, organizationName, maxResults) if err != nil { - log.Fatalf("Error fetching organization repositories: %v", err) + return nil, fmt.Errorf("error fetching organization repositories: %w", err) } - var values []models.Resource + // Filter repositories based on excludeArchived and excludeDisabled + var filteredRepos []model.MinimalRepoInfo for _, r := range allRepos { - value := getRepositoriesDetail(ctx, sdk, organizationName, r.Name, stream) - values = append(values, *value) + if excludeArchived && r.Archived { + continue + } + if excludeDisabled && r.Disabled { + continue + } + filteredRepos = append(filteredRepos, r) + } + + // Multi-threading (5 workers) for fetching repository details + concurrency := 5 + results := make([]models.Resource, len(filteredRepos)) + + type job struct { + index int + repo string + } + + jobCh := make(chan job) + wg := sync.WaitGroup{} + + worker := func() { + defer wg.Done() + for j := range jobCh { + value := _getRepositoriesDetail(ctx, sdk, organizationName, j.repo, stream) + if value != nil { + results[j.index] = *value + } + } + } + + // Start workers + for i := 0; i < concurrency; i++ { + wg.Add(1) + go worker() + } + + // Send jobs + for i, r := range filteredRepos { + jobCh <- job{index: i, repo: r.Name} + } + close(jobCh) + + // Wait for all workers to finish + wg.Wait() + + // Filter out empty results in case some fetches failed + var finalResults []models.Resource + for _, res := range results { + if res.ID != "" { + finalResults = append(finalResults, res) + } } - return values, nil + return finalResults, nil } -func getRepositoriesDetail(ctx context.Context, sdk *resilientbridge.ResilientBridge, organizationName, repo string, stream *models.StreamSender) *models.Resource { - repoDetail, err := fetchRepoDetails(sdk, organizationName, repo) +// GetRepositoryDetails returns details for a given repo +func GetRepositoryDetails(ctx context.Context, githubClient GitHubClient, organizationName, repositoryName string) (*models.Resource, error) { + sdk := resilientbridge.NewResilientBridge() + sdk.RegisterProvider("github", adapters.NewGitHubAdapter(githubClient.Token), &resilientbridge.ProviderConfig{ + UseProviderLimits: true, + MaxRetries: 3, + BaseBackoff: 0, + }) + + repoDetail, err := _fetchRepoDetails(sdk, organizationName, repositoryName) if err != nil { - log.Printf("Error fetching details for %s/%s: %v", organizationName, repo, err) - return nil + return nil, fmt.Errorf("error fetching repository details for %s/%s: %w", organizationName, repositoryName, err) } - finalDetail := transformToFinalRepoDetail(repoDetail) - // Fetch languages - langs, err := fetchLanguages(sdk, organizationName, repo) + finalDetail := _transformToFinalRepoDetail(repoDetail) + + langs, err := _fetchLanguages(sdk, organizationName, repositoryName) if err == nil { finalDetail.Languages = langs } - // Enrich metrics - err = enrichRepoMetrics(sdk, organizationName, repo, finalDetail) + err = _enrichRepoMetrics(sdk, organizationName, repositoryName, finalDetail) if err != nil { - log.Printf("Error enriching repo metrics for %s/%s: %v", organizationName, repo, err) - } - - // **New addition: Fetch private vulnerability reporting status** - pvrEnabled, err := fetchPrivateVulnerabilityReporting(sdk, organizationName, repo) - if err != nil { - log.Printf("Error fetching private vulnerability reporting status for %s/%s: %v", organizationName, repo, err) - } else { - finalDetail.SecuritySettings.PrivateVulnerabilityReportingEnabled = pvrEnabled + log.Printf("Error enriching repo metrics for %s/%s: %v", organizationName, repositoryName, err) } value := models.Resource{ ID: strconv.Itoa(finalDetail.GitHubRepoID), Name: finalDetail.Name, Description: JSONAllFieldsMarshaller{ - Value: model.RepositoryDescription{ - GitHubRepoID: finalDetail.GitHubRepoID, - NodeID: finalDetail.NodeID, - Name: finalDetail.Name, - NameWithOwner: finalDetail.NameWithOwner, - Description: finalDetail.Description, - CreatedAt: finalDetail.CreatedAt, - UpdatedAt: finalDetail.UpdatedAt, - PushedAt: finalDetail.PushedAt, - IsActive: finalDetail.IsActive, - IsEmpty: finalDetail.IsEmpty, - IsFork: finalDetail.IsFork, - IsSecurityPolicyEnabled: finalDetail.IsSecurityPolicyEnabled, - Owner: finalDetail.Owner, - HomepageURL: finalDetail.HomepageURL, - LicenseInfo: finalDetail.LicenseInfo, - Topics: finalDetail.Topics, - Visibility: finalDetail.Visibility, - DefaultBranchRef: finalDetail.DefaultBranchRef, - Permissions: finalDetail.Permissions, - Organization: finalDetail.Organization, - Parent: finalDetail.Parent, - Source: finalDetail.Source, - Languages: finalDetail.Languages, - RepositorySettings: finalDetail.RepositorySettings, - SecuritySettings: finalDetail.SecuritySettings, - RepoURLs: finalDetail.RepoURLs, - Metrics: finalDetail.Metrics, - }, + Value: finalDetail, }, } - if stream != nil { - if err := (*stream)(value); err != nil { - return nil - } - } - return &value + return &value, nil } -//func GetRepositoryList(ctx context.Context, githubClient GitHubClient, organizationName string, stream *models.StreamSender) ([]models.Resource, error) { -// client := githubClient.GraphQLClient -// query := struct { -// RateLimit steampipemodels.RateLimit -// Organization struct { -// Repositories struct { -// PageInfo steampipemodels.PageInfo -// TotalCount int -// Nodes []steampipemodels.Repository -// } `graphql:"repositories(first: $pageSize, after: $cursor)"` -// } `graphql:"organization(login: $owner)"` // <-- $owner used here -// }{} -// variables := map[string]interface{}{ -// "owner": githubv4.String(organizationName), -// "pageSize": githubv4.Int(repoPageSize), -// "cursor": (*githubv4.String)(nil), -// } -// columnNames := repositoryCols() -// appendRepoColumnIncludes(&variables, columnNames) -// var values []models.Resource -// for { -// err := client.Query(ctx, &query, variables) -// if err != nil { -// return nil, err -// } -// for _, repo := range query.Organization.Repositories.Nodes { -// hooks, err := GetRepositoryHooks(ctx, githubClient.RestClient, organizationName, repo.Name) -// if err != nil { -// return nil, err -// } -// additionalRepoInfo, err := GetRepositoryAdditionalData(ctx, githubClient.RestClient, organizationName, repo.Name) -// value := models.Resource{ -// ID: strconv.Itoa(repo.Id), -// Name: repo.Name, -// Description: JSONAllFieldsMarshaller{ -// Value: model.RepositoryDescription{ -// ID: repo.Id, -// NodeID: repo.NodeId, -// Name: repo.Name, -// AllowUpdateBranch: repo.AllowUpdateBranch, -// ArchivedAt: repo.ArchivedAt, -// AutoMergeAllowed: repo.AutoMergeAllowed, -// CodeOfConduct: repo.CodeOfConduct, -// ContactLinks: repo.ContactLinks, -// CreatedAt: repo.CreatedAt, -// DefaultBranchRef: repo.DefaultBranchRef, -// DeleteBranchOnMerge: repo.DeleteBranchOnMerge, -// Description: repo.Description, -// DiskUsage: repo.DiskUsage, -// ForkCount: repo.ForkCount, -// ForkingAllowed: repo.ForkingAllowed, -// FundingLinks: repo.FundingLinks, -// HasDiscussionsEnabled: repo.HasDiscussionsEnabled, -// HasIssuesEnabled: repo.HasIssuesEnabled, -// HasProjectsEnabled: repo.HasProjectsEnabled, -// HasVulnerabilityAlertsEnabled: repo.HasVulnerabilityAlertsEnabled, -// HasWikiEnabled: repo.HasWikiEnabled, -// HomepageURL: repo.HomepageUrl, -// InteractionAbility: repo.InteractionAbility, -// IsArchived: repo.IsArchived, -// IsBlankIssuesEnabled: repo.IsBlankIssuesEnabled, -// IsDisabled: repo.IsDisabled, -// IsEmpty: repo.IsEmpty, -// IsFork: repo.IsFork, -// IsInOrganization: repo.IsInOrganization, -// IsLocked: repo.IsLocked, -// IsMirror: repo.IsMirror, -// IsPrivate: repo.IsPrivate, -// IsSecurityPolicyEnabled: repo.IsSecurityPolicyEnabled, -// IsTemplate: repo.IsTemplate, -// IsUserConfigurationRepository: repo.IsUserConfigurationRepository, -// IssueTemplates: repo.IssueTemplates, -// LicenseInfo: repo.LicenseInfo, -// LockReason: repo.LockReason, -// MergeCommitAllowed: repo.MergeCommitAllowed, -// MergeCommitMessage: repo.MergeCommitMessage, -// MergeCommitTitle: repo.MergeCommitTitle, -// MirrorURL: repo.MirrorUrl, -// NameWithOwner: repo.NameWithOwner, -// OpenGraphImageURL: repo.OpenGraphImageUrl, -// OwnerLogin: repo.Owner.Login, -// PrimaryLanguage: repo.PrimaryLanguage, -// ProjectsURL: repo.ProjectsUrl, -// PullRequestTemplates: repo.PullRequestTemplates, -// PushedAt: repo.PushedAt, -// RebaseMergeAllowed: repo.RebaseMergeAllowed, -// SecurityPolicyURL: repo.SecurityPolicyUrl, -// SquashMergeAllowed: repo.SquashMergeAllowed, -// SquashMergeCommitMessage: repo.SquashMergeCommitMessage, -// SquashMergeCommitTitle: repo.SquashMergeCommitTitle, -// SSHURL: repo.SshUrl, -// StargazerCount: repo.StargazerCount, -// UpdatedAt: repo.UpdatedAt, -// URL: repo.Url, -// // UsesCustomOpenGraphImage: repo.UsesCustomOpenGraphImage, -// // CanAdminister: repo.CanAdminister, -// // CanCreateProjects: repo.CanCreateProjects, -// // CanSubscribe: repo.CanSubscribe, -// // CanUpdateTopics: repo.CanUpdateTopics, -// // HasStarred: repo.HasStarred, -// PossibleCommitEmails: repo.PossibleCommitEmails, -// // Subscription: repo.Subscription, -// Visibility: repo.Visibility, -// // YourPermission: repo.YourPermission, -// WebCommitSignOffRequired: repo.WebCommitSignoffRequired, -// RepositoryTopicsTotalCount: repo.RepositoryTopics.TotalCount, -// OpenIssuesTotalCount: repo.OpenIssues.TotalCount, -// WatchersTotalCount: repo.Watchers.TotalCount, -// Hooks: hooks, -// Topics: additionalRepoInfo.Topics, -// SubscribersCount: additionalRepoInfo.GetSubscribersCount(), -// HasDownloads: additionalRepoInfo.GetHasDownloads(), -// HasPages: additionalRepoInfo.GetHasPages(), -// NetworkCount: additionalRepoInfo.GetNetworkCount(), -// }, -// }, -// } -// if stream != nil { -// if err := (*stream)(value); err != nil { -// return nil, err -// } -// } else { -// values = append(values, value) -// } -// } -// if !query.Organization.Repositories.PageInfo.HasNextPage { -// break -// } -// variables["cursor"] = githubv4.NewString(query.Organization.Repositories.PageInfo.EndCursor) -// } -// return values, nil -//} - -//func GetRepository(ctx context.Context, githubClient GitHubClient, organizationName string, repositoryName string, resourceID string, stream *models.StreamSender) (*models.Resource, error) { -// client := githubClient.GraphQLClient -// query := struct { -// RateLimit steampipemodels.RateLimit -// Organization struct { -// Repository steampipemodels.Repository `graphql:"repository(name: $repoName)"` -// } `graphql:"organization(login: $owner)"` // <-- $owner used here -// }{} -// -// variables := map[string]interface{}{ -// "owner": githubv4.String(organizationName), -// "repoName": githubv4.String(repositoryName), -// } -// -// columnNames := repositoryCols() -// appendRepoColumnIncludes(&variables, columnNames) -// err := client.Query(ctx, &query, variables) -// if err != nil { -// return nil, err -// } -// repo := query.Organization.Repository -// hooks, err := GetRepositoryHooks(ctx, githubClient.RestClient, organizationName, repo.Name) -// if err != nil { -// return nil, err -// } -// additionalRepoInfo, err := GetRepositoryAdditionalData(ctx, githubClient.RestClient, organizationName, repo.Name) -// value := models.Resource{ -// ID: strconv.Itoa(repo.Id), -// Name: repo.Name, -// Description: JSONAllFieldsMarshaller{ -// Value: model.RepositoryDescription{ -// ID: repo.Id, -// NodeID: repo.NodeId, -// Name: repo.Name, -// AllowUpdateBranch: repo.AllowUpdateBranch, -// ArchivedAt: repo.ArchivedAt, -// AutoMergeAllowed: repo.AutoMergeAllowed, -// CodeOfConduct: repo.CodeOfConduct, -// ContactLinks: repo.ContactLinks, -// CreatedAt: repo.CreatedAt, -// DefaultBranchRef: repo.DefaultBranchRef, -// DeleteBranchOnMerge: repo.DeleteBranchOnMerge, -// Description: repo.Description, -// DiskUsage: repo.DiskUsage, -// ForkCount: repo.ForkCount, -// ForkingAllowed: repo.ForkingAllowed, -// FundingLinks: repo.FundingLinks, -// HasDiscussionsEnabled: repo.HasDiscussionsEnabled, -// HasIssuesEnabled: repo.HasIssuesEnabled, -// HasProjectsEnabled: repo.HasProjectsEnabled, -// HasVulnerabilityAlertsEnabled: repo.HasVulnerabilityAlertsEnabled, -// HasWikiEnabled: repo.HasWikiEnabled, -// HomepageURL: repo.HomepageUrl, -// InteractionAbility: repo.InteractionAbility, -// IsArchived: repo.IsArchived, -// IsBlankIssuesEnabled: repo.IsBlankIssuesEnabled, -// IsDisabled: repo.IsDisabled, -// IsEmpty: repo.IsEmpty, -// IsFork: repo.IsFork, -// IsInOrganization: repo.IsInOrganization, -// IsLocked: repo.IsLocked, -// IsMirror: repo.IsMirror, -// IsPrivate: repo.IsPrivate, -// IsSecurityPolicyEnabled: repo.IsSecurityPolicyEnabled, -// IsTemplate: repo.IsTemplate, -// IsUserConfigurationRepository: repo.IsUserConfigurationRepository, -// IssueTemplates: repo.IssueTemplates, -// LicenseInfo: repo.LicenseInfo, -// LockReason: repo.LockReason, -// MergeCommitAllowed: repo.MergeCommitAllowed, -// MergeCommitMessage: repo.MergeCommitMessage, -// MergeCommitTitle: repo.MergeCommitTitle, -// MirrorURL: repo.MirrorUrl, -// NameWithOwner: repo.NameWithOwner, -// OpenGraphImageURL: repo.OpenGraphImageUrl, -// OwnerLogin: repo.Owner.Login, -// PrimaryLanguage: repo.PrimaryLanguage, -// ProjectsURL: repo.ProjectsUrl, -// PullRequestTemplates: repo.PullRequestTemplates, -// PushedAt: repo.PushedAt, -// RebaseMergeAllowed: repo.RebaseMergeAllowed, -// SecurityPolicyURL: repo.SecurityPolicyUrl, -// SquashMergeAllowed: repo.SquashMergeAllowed, -// SquashMergeCommitMessage: repo.SquashMergeCommitMessage, -// SquashMergeCommitTitle: repo.SquashMergeCommitTitle, -// SSHURL: repo.SshUrl, -// StargazerCount: repo.StargazerCount, -// UpdatedAt: repo.UpdatedAt, -// URL: repo.Url, -// // UsesCustomOpenGraphImage: repo.UsesCustomOpenGraphImage, -// // CanAdminister: repo.CanAdminister, -// // CanCreateProjects: repo.CanCreateProjects, -// // CanSubscribe: repo.CanSubscribe, -// // CanUpdateTopics: repo.CanUpdateTopics, -// // HasStarred: repo.HasStarred, -// PossibleCommitEmails: repo.PossibleCommitEmails, -// // Subscription: repo.Subscription, -// Visibility: repo.Visibility, -// // YourPermission: repo.YourPermission, -// WebCommitSignOffRequired: repo.WebCommitSignoffRequired, -// RepositoryTopicsTotalCount: repo.RepositoryTopics.TotalCount, -// OpenIssuesTotalCount: repo.OpenIssues.TotalCount, -// WatchersTotalCount: repo.Watchers.TotalCount, -// Hooks: hooks, -// Topics: additionalRepoInfo.Topics, -// SubscribersCount: additionalRepoInfo.GetSubscribersCount(), -// HasDownloads: additionalRepoInfo.GetHasDownloads(), -// HasPages: additionalRepoInfo.GetHasPages(), -// NetworkCount: additionalRepoInfo.GetNetworkCount(), -// }, -// }, -// } -// if stream != nil { -// if err := (*stream)(value); err != nil { -// return nil, err -// } -// } -// -// return &value, nil -//} - -func GetRepositoryAdditionalData(ctx context.Context, client *github.Client, organizationName string, repo string) (*github.Repository, error) { - repository, _, err := client.Repositories.Get(ctx, organizationName, repo) +// Utility/helper functions (prefixed with underscore) + +// _fetchLanguages fetches repository languages. +func _fetchLanguages(sdk *resilientbridge.ResilientBridge, owner, repo string) (map[string]int, error) { + req := &resilientbridge.NormalizedRequest{ + Method: "GET", + Endpoint: fmt.Sprintf("/repos/%s/%s/languages", owner, repo), + Headers: map[string]string{"Accept": "application/vnd.github+json"}, + } + + resp, err := sdk.Request("github", req) if err != nil { - if strings.Contains(err.Error(), "404") { - return nil, nil - } - return nil, nil + return nil, fmt.Errorf("error fetching languages: %w", err) } - if repository == nil { - return nil, nil + + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) } - return repository, nil -} -func GetRepositoryHooks(ctx context.Context, client *github.Client, organizationName string, repo string) ([]*github.Hook, error) { - var repositoryHooks []*github.Hook - opt := &github.ListOptions{PerPage: pageSize} - for { - hooks, resp, err := client.Repositories.ListHooks(ctx, organizationName, repo, opt) - if err != nil && strings.Contains(err.Error(), "Not Found") { - return nil, nil - } else if err != nil { - return nil, err - } - repositoryHooks = append(repositoryHooks, hooks...) - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage + var langs map[string]int + if err := json.Unmarshal(resp.Data, &langs); err != nil { + return nil, fmt.Errorf("error decoding languages: %w", err) } - return repositoryHooks, nil + return langs, nil } -func enrichRepoMetrics(sdk *resilientbridge.ResilientBridge, owner, repoName string, finalDetail *model.RepositoryDescription) error { +// _enrichRepoMetrics enriches repo metrics such as commits, issues, branches, etc. +func _enrichRepoMetrics(sdk *resilientbridge.ResilientBridge, owner, repoName string, finalDetail *model.RepositoryDescription) error { var dbObj map[string]string if finalDetail.DefaultBranchRef != nil { if err := json.Unmarshal(finalDetail.DefaultBranchRef, &dbObj); err != nil { @@ -409,129 +174,137 @@ func enrichRepoMetrics(sdk *resilientbridge.ResilientBridge, owner, repoName str defaultBranch = "main" } - commitsCount, err := countCommits(sdk, owner, repoName, defaultBranch) + commitsCount, err := _countCommits(sdk, owner, repoName, defaultBranch) if err != nil { return fmt.Errorf("counting commits: %w", err) } finalDetail.Metrics.Commits = commitsCount - issuesCount, err := countIssues(sdk, owner, repoName) + issuesCount, err := _countIssues(sdk, owner, repoName) if err != nil { return fmt.Errorf("counting issues: %w", err) } finalDetail.Metrics.Issues = issuesCount - branchesCount, err := countBranches(sdk, owner, repoName) + branchesCount, err := _countBranches(sdk, owner, repoName) if err != nil { return fmt.Errorf("counting branches: %w", err) } finalDetail.Metrics.Branches = branchesCount - prCount, err := countPullRequests(sdk, owner, repoName) + prCount, err := _countPullRequests(sdk, owner, repoName) if err != nil { return fmt.Errorf("counting PRs: %w", err) } finalDetail.Metrics.PullRequests = prCount - releasesCount, err := countReleases(sdk, owner, repoName) + releasesCount, err := _countReleases(sdk, owner, repoName) if err != nil { return fmt.Errorf("counting releases: %w", err) } finalDetail.Metrics.Releases = releasesCount - // New: Count tags - tagsCount, err := countTags(sdk, owner, repoName) + tagsCount, err := _countTags(sdk, owner, repoName) if err != nil { return fmt.Errorf("counting tags: %w", err) } - // Add "TotalTags" field to RepoMetrics struct and assign here - // You need to add the `TotalTags int `json:"total_tags"` field to RepoMetrics beforehand finalDetail.Metrics.Tags = tagsCount return nil } -// New function countTags -func countTags(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { +func _countTags(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { endpoint := fmt.Sprintf("/repos/%s/%s/tags?per_page=1", owner, repoName) - return countItemsFromEndpoint(sdk, endpoint) + return _countItemsFromEndpoint(sdk, endpoint) } -func fetchLanguages(sdk *resilientbridge.ResilientBridge, owner, repo string) (map[string]int, error) { - req := &resilientbridge.NormalizedRequest{ - Method: "GET", - Endpoint: fmt.Sprintf("/repos/%s/%s/languages", owner, repo), - Headers: map[string]string{"Accept": "application/vnd.github+json"}, - } - - resp, err := sdk.Request("github", req) - if err != nil { - return nil, fmt.Errorf("error fetching languages: %w", err) - } +func _countCommits(sdk *resilientbridge.ResilientBridge, owner, repoName, defaultBranch string) (int, error) { + endpoint := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&per_page=1", owner, repoName, defaultBranch) + return _countItemsFromEndpoint(sdk, endpoint) +} - if resp.StatusCode >= 400 { - return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) - } +func _countIssues(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { + endpoint := fmt.Sprintf("/repos/%s/%s/issues?state=all&per_page=1", owner, repoName) + return _countItemsFromEndpoint(sdk, endpoint) +} - var langs map[string]int - if err := json.Unmarshal(resp.Data, &langs); err != nil { - return nil, fmt.Errorf("error decoding languages: %w", err) - } - return langs, nil +func _countBranches(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { + endpoint := fmt.Sprintf("/repos/%s/%s/branches?per_page=1", owner, repoName) + return _countItemsFromEndpoint(sdk, endpoint) } -// The rest of the functions (parseScopeURL, fetchOrgRepos, fetchRepoDetails, transformToFinalRepoDetail, -// enrichRepoMetrics, countCommits, countIssues, countBranches, countPullRequests, countReleases, -// countItemsFromEndpoint, and parseLastPage) remain unchanged. +func _countPullRequests(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { + endpoint := fmt.Sprintf("/repos/%s/%s/pulls?state=all&per_page=1", owner, repoName) + return _countItemsFromEndpoint(sdk, endpoint) +} -func parseScopeURL(repoURL string) (owner, repo string, err error) { - if !strings.HasPrefix(repoURL, "https://github.com/") { - return "", "", fmt.Errorf("URL must start with https://github.com/") - } - parts := strings.Split(strings.TrimPrefix(repoURL, "https://github.com/"), "/") - if len(parts) == 0 || parts[0] == "" { - return "", "", fmt.Errorf("invalid URL format") - } - owner = parts[0] - if len(parts) > 1 { - repo = parts[1] - } - return owner, repo, nil +func _countReleases(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { + endpoint := fmt.Sprintf("/repos/%s/%s/releases?per_page=1", owner, repoName) + return _countItemsFromEndpoint(sdk, endpoint) } -func fetchPrivateVulnerabilityReporting(sdk *resilientbridge.ResilientBridge, owner, repoName string) (bool, error) { +func _countItemsFromEndpoint(sdk *resilientbridge.ResilientBridge, endpoint string) (int, error) { req := &resilientbridge.NormalizedRequest{ Method: "GET", - Endpoint: fmt.Sprintf("/repos/%s/%s/private-vulnerability-reporting", owner, repoName), + Endpoint: endpoint, Headers: map[string]string{"Accept": "application/vnd.github+json"}, } resp, err := sdk.Request("github", req) if err != nil { - return false, fmt.Errorf("error fetching private vulnerability reporting: %w", err) + return 0, fmt.Errorf("error fetching data: %w", err) } - if resp.StatusCode == 404 { - // Endpoint returns 404 if private vulnerability reporting is not enabled - // or the resource is not found. Default to false. - return false, nil + if resp.StatusCode == 409 { + return 0, nil } if resp.StatusCode >= 400 { - return false, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) + return 0, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) + } + + var linkHeader string + for k, v := range resp.Headers { + if strings.ToLower(k) == "link" { + linkHeader = v + break + } } - var result struct { - Enabled bool `json:"enabled"` + if linkHeader == "" { + if len(resp.Data) > 2 { + var items []interface{} + if err := json.Unmarshal(resp.Data, &items); err != nil { + return 1, nil + } + return len(items), nil + } + return 0, nil } - if err := json.Unmarshal(resp.Data, &result); err != nil { - return false, fmt.Errorf("error decoding private vulnerability reporting status: %w", err) + + lastPage, err := _parseLastPage(linkHeader) + if err != nil { + return 0, fmt.Errorf("could not parse last page: %w", err) } - return result.Enabled, nil + return lastPage, nil +} + +func _parseLastPage(linkHeader string) (int, error) { + re := regexp.MustCompile(`page=(\d+)>; rel="last"`) + matches := re.FindStringSubmatch(linkHeader) + if len(matches) < 2 { + return 1, nil + } + var lastPage int + _, err := fmt.Sscanf(matches[1], "%d", &lastPage) + if err != nil { + return 0, err + } + return lastPage, nil } -func fetchOrgRepos(sdk *resilientbridge.ResilientBridge, org string, maxResults int) ([]model.MinimalRepoInfo, error) { +func _fetchOrgRepos(sdk *resilientbridge.ResilientBridge, org string, maxResults int) ([]model.MinimalRepoInfo, error) { var allRepos []model.MinimalRepoInfo perPage := 100 page := 1 @@ -579,7 +352,7 @@ func fetchOrgRepos(sdk *resilientbridge.ResilientBridge, org string, maxResults return allRepos, nil } -func fetchRepoDetails(sdk *resilientbridge.ResilientBridge, owner, repo string) (*model.RepoDetail, error) { +func _fetchRepoDetails(sdk *resilientbridge.ResilientBridge, owner, repo string) (*model.RepoDetail, error) { req := &resilientbridge.NormalizedRequest{ Method: "GET", Endpoint: fmt.Sprintf("/repos/%s/%s", owner, repo), @@ -600,17 +373,16 @@ func fetchRepoDetails(sdk *resilientbridge.ResilientBridge, owner, repo string) return &detail, nil } -func transformToFinalRepoDetail(detail *model.RepoDetail) *model.RepositoryDescription { +func _transformToFinalRepoDetail(detail *model.RepoDetail) *model.RepositoryDescription { var parent *model.RepositoryDescription if detail.Parent != nil { - parent = transformToFinalRepoDetail(detail.Parent) + parent = _transformToFinalRepoDetail(detail.Parent) } var source *model.RepositoryDescription if detail.Source != nil { - source = transformToFinalRepoDetail(detail.Source) + source = _transformToFinalRepoDetail(detail.Source) } - // Owner: no user_view_type, no site_admin var finalOwner *model.Owner if detail.Owner != nil { finalOwner = &model.Owner{ @@ -622,7 +394,6 @@ func transformToFinalRepoDetail(detail *model.RepoDetail) *model.RepositoryDescr } } - // Organization: includes user_view_type, site_admin var finalOrg *model.Organization if detail.Organization != nil { finalOrg = &model.Organization{ @@ -636,14 +407,10 @@ func transformToFinalRepoDetail(detail *model.RepoDetail) *model.RepositoryDescr } } - // Prepare default_branch_ref as before dbObj := map[string]string{"name": detail.DefaultBranch} dbBytes, _ := json.Marshal(dbObj) - // Determine is_active: true if not archived and not disabled isActive := !(detail.Archived || detail.Disabled) - //isInOrganization := (detail.Organization != nil && detail.Organization.Type == "Organization") - //isMirror := (detail.MirrorURL != nil) isEmpty := (detail.Size == 0) var licenseJSON json.RawMessage @@ -653,32 +420,29 @@ func transformToFinalRepoDetail(detail *model.RepoDetail) *model.RepositoryDescr } finalDetail := &model.RepositoryDescription{ - GitHubRepoID: detail.ID, - NodeID: detail.NodeID, - Name: detail.Name, - NameWithOwner: detail.FullName, - Description: detail.Description, - CreatedAt: detail.CreatedAt, - UpdatedAt: detail.UpdatedAt, - PushedAt: detail.PushedAt, - IsActive: isActive, - IsEmpty: isEmpty, - IsFork: detail.Fork, - //IsInOrganization: isInOrganization, - //IsMirror: isMirror, - IsSecurityPolicyEnabled: false, // as before - //IsTemplate: detail.IsTemplate, - Owner: finalOwner, - HomepageURL: detail.Homepage, - LicenseInfo: licenseJSON, - Topics: detail.Topics, - Visibility: detail.Visibility, - DefaultBranchRef: dbBytes, - Permissions: detail.Permissions, - Organization: finalOrg, - Parent: parent, - Source: source, - Languages: nil, // set after fetchLanguages + GitHubRepoID: detail.ID, + NodeID: detail.NodeID, + Name: detail.Name, + NameWithOwner: detail.FullName, + Description: detail.Description, + CreatedAt: detail.CreatedAt, + UpdatedAt: detail.UpdatedAt, + PushedAt: detail.PushedAt, + IsActive: isActive, + IsEmpty: isEmpty, + IsFork: detail.Fork, + IsSecurityPolicyEnabled: false, + Owner: finalOwner, + HomepageURL: detail.Homepage, + LicenseInfo: licenseJSON, + Topics: detail.Topics, + Visibility: detail.Visibility, + DefaultBranchRef: dbBytes, + Permissions: detail.Permissions, + Organization: finalOrg, + Parent: parent, + Source: source, + Languages: nil, RepositorySettings: model.RepositorySettings{ HasDiscussionsEnabled: detail.HasDiscussions, HasIssuesEnabled: detail.HasIssues, @@ -713,7 +477,6 @@ func transformToFinalRepoDetail(detail *model.RepoDetail) *model.RepositoryDescr DependabotSecurityUpdatesEnabled: false, SecretScanningNonProviderPatternsEnabled: false, SecretScanningValidityChecksEnabled: false, - PrivateVulnerabilityReportingEnabled: false, }, RepoURLs: model.RepoURLs{ GitURL: detail.GitURL, @@ -727,99 +490,41 @@ func transformToFinalRepoDetail(detail *model.RepoDetail) *model.RepositoryDescr Forks: detail.ForksCount, Subscribers: detail.SubscribersCount, Size: detail.Size, - // The rest (tags, commits, issues, open_issues, branches, pull_requests, releases) - // will be set after calling enrichRepoMetrics and assigning open issues from detail. + OpenIssues: detail.OpenIssuesCount, }, } - - // Set open_issues before enrichRepoMetrics if needed: - finalDetail.Metrics.OpenIssues = detail.OpenIssuesCount - return finalDetail } -func countCommits(sdk *resilientbridge.ResilientBridge, owner, repoName, defaultBranch string) (int, error) { - endpoint := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&per_page=1", owner, repoName, defaultBranch) - return countItemsFromEndpoint(sdk, endpoint) -} - -func countIssues(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { - endpoint := fmt.Sprintf("/repos/%s/%s/issues?state=all&per_page=1", owner, repoName) - return countItemsFromEndpoint(sdk, endpoint) -} - -func countBranches(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { - endpoint := fmt.Sprintf("/repos/%s/%s/branches?per_page=1", owner, repoName) - return countItemsFromEndpoint(sdk, endpoint) -} - -func countPullRequests(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { - endpoint := fmt.Sprintf("/repos/%s/%s/pulls?state=all&per_page=1", owner, repoName) - return countItemsFromEndpoint(sdk, endpoint) -} - -func countReleases(sdk *resilientbridge.ResilientBridge, owner, repoName string) (int, error) { - endpoint := fmt.Sprintf("/repos/%s/%s/releases?per_page=1", owner, repoName) - return countItemsFromEndpoint(sdk, endpoint) -} - -func countItemsFromEndpoint(sdk *resilientbridge.ResilientBridge, endpoint string) (int, error) { - req := &resilientbridge.NormalizedRequest{ - Method: "GET", - Endpoint: endpoint, - Headers: map[string]string{"Accept": "application/vnd.github+json"}, - } - - resp, err := sdk.Request("github", req) +func _getRepositoriesDetail(ctx context.Context, sdk *resilientbridge.ResilientBridge, organizationName, repo string, stream *models.StreamSender) *models.Resource { + repoDetail, err := _fetchRepoDetails(sdk, organizationName, repo) if err != nil { - return 0, fmt.Errorf("error fetching data: %w", err) - } - - if resp.StatusCode == 409 { - return 0, nil - } - - if resp.StatusCode >= 400 { - return 0, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, string(resp.Data)) - } - - var linkHeader string - for k, v := range resp.Headers { - if strings.ToLower(k) == "link" { - linkHeader = v - break - } + log.Printf("Error fetching details for %s/%s: %v", organizationName, repo, err) + return nil } - if linkHeader == "" { - if len(resp.Data) > 2 { - var items []interface{} - if err := json.Unmarshal(resp.Data, &items); err != nil { - return 1, nil - } - return len(items), nil - } - return 0, nil + finalDetail := _transformToFinalRepoDetail(repoDetail) + langs, err := _fetchLanguages(sdk, organizationName, repo) + if err == nil { + finalDetail.Languages = langs } - lastPage, err := parseLastPage(linkHeader) + err = _enrichRepoMetrics(sdk, organizationName, repo, finalDetail) if err != nil { - return 0, fmt.Errorf("could not parse last page: %w", err) + log.Printf("Error enriching repo metrics for %s/%s: %v", organizationName, repo, err) } - return lastPage, nil -} - -func parseLastPage(linkHeader string) (int, error) { - re := regexp.MustCompile(`page=(\d+)>; rel="last"`) - matches := re.FindStringSubmatch(linkHeader) - if len(matches) < 2 { - return 1, nil + value := models.Resource{ + ID: strconv.Itoa(finalDetail.GitHubRepoID), + Name: finalDetail.Name, + Description: JSONAllFieldsMarshaller{ + Value: finalDetail, + }, } - var lastPage int - _, err := fmt.Sscanf(matches[1], "%d", &lastPage) - if err != nil { - return 0, err + if stream != nil { + if err := (*stream)(value); err != nil { + return nil + } } - return lastPage, nil + return &value } diff --git a/provider/model/model.go b/provider/model/model.go index ed655ec6..48474745 100755 --- a/provider/model/model.go +++ b/provider/model/model.go @@ -6,11 +6,12 @@ package model import ( "encoding/json" + "time" + goPipeline "github.com/buildkite/go-pipeline" "github.com/google/go-github/v55/github" "github.com/shurcooL/githubv4" steampipemodels "github.com/turbot/steampipe-plugin-github/github/models" - "time" ) type Metadata struct{} @@ -681,8 +682,10 @@ type RepositoryDescription struct { } type MinimalRepoInfo struct { - Name string `json:"name"` - Owner struct { + Name string `json:"name"` + Archived bool `json:"archived"` + Disabled bool `json:"disabled"` + Owner struct { Login string `json:"login"` } `json:"owner"` }