From 45fc2f8151a2a4e1e736081d2c7dc080100a4043 Mon Sep 17 00:00:00 2001 From: artaasadi Date: Wed, 20 Nov 2024 21:40:29 +0100 Subject: [PATCH] fix: fix page sizes --- command/cmd/describer.go | 26 +++-- command/cmd/getDescriber.go | 25 ++-- provider/describer/release.go | 2 +- provider/describer/repository.go | 121 ++++++++++++++++++++ provider/describer/utils.go | 5 +- provider/resource_types.go | 2 +- provider/resource_types/resource-types.json | 2 +- 7 files changed, 158 insertions(+), 25 deletions(-) diff --git a/command/cmd/describer.go b/command/cmd/describer.go index 350a90d1..8068ff45 100644 --- a/command/cmd/describer.go +++ b/command/cmd/describer.go @@ -38,22 +38,28 @@ var describerCmd = &cobra.Command{ defer file.Close() // Ensure the file is closed at the end job := describe.DescribeJob{ - JobID: uint(uuid.New().ID()), - ResourceType: resourceType, - IntegrationID: "", - ProviderID: "", - DescribedAt: time.Now().UnixMilli(), - IntegrationType: configs.IntegrationTypeLower, - CipherText: "", - IntegrationLabels: nil, + JobID: uint(uuid.New().ID()), + ResourceType: resourceType, + IntegrationID: "", + ProviderID: "", + DescribedAt: time.Now().UnixMilli(), + IntegrationType: configs.IntegrationTypeLower, + CipherText: "", + IntegrationLabels: map[string]string{ + "OrganizationName": OrganizationName, + }, IntegrationAnnotations: nil, } ctx := context.Background() logger, _ := zap.NewProduction() - // TODO: Set the credentials - creds := configs.IntegrationCredentials{} + creds, err := provider.AccountCredentialsFromMap(map[string]any{ + "pat_token": PatToken, + }) + if err != nil { + return fmt.Errorf(" account credentials: %w", err) + } additionalParameters, err := provider.GetAdditionalParameters(job) if err != nil { diff --git a/command/cmd/getDescriber.go b/command/cmd/getDescriber.go index 35ef690d..1bf00594 100644 --- a/command/cmd/getDescriber.go +++ b/command/cmd/getDescriber.go @@ -21,8 +21,10 @@ import ( ) var ( - resourceID string - PatToken = os.Getenv("PAT_TOKEN") + resourceID string + PatToken = os.Getenv("PAT_TOKEN") + RepositoryName = os.Getenv("REPOSITORY_NAME") + OrganizationName = os.Getenv("ORGANIZATION_NAME") ) // getDescriberCmd represents the describer command @@ -38,14 +40,16 @@ var getDescriberCmd = &cobra.Command{ defer file.Close() // Ensure the file is closed at the end job := describe.DescribeJob{ - JobID: uint(uuid.New().ID()), - ResourceType: resourceType, - IntegrationID: "", - ProviderID: "", - DescribedAt: time.Now().UnixMilli(), - IntegrationType: configs.IntegrationTypeLower, - CipherText: "", - IntegrationLabels: nil, + JobID: uint(uuid.New().ID()), + ResourceType: resourceType, + IntegrationID: "", + ProviderID: "", + DescribedAt: time.Now().UnixMilli(), + IntegrationType: configs.IntegrationTypeLower, + CipherText: "", + IntegrationLabels: map[string]string{ + "OrganizationName": OrganizationName, + }, IntegrationAnnotations: nil, } @@ -64,6 +68,7 @@ var getDescriberCmd = &cobra.Command{ return err } plg := steampipe.Plugin() + additionalParameters["RepositoryName"] = RepositoryName f := func(resource model.Resource) error { if resource.Description == nil { diff --git a/provider/describer/release.go b/provider/describer/release.go index a2b02c62..6093b92d 100644 --- a/provider/describer/release.go +++ b/provider/describer/release.go @@ -15,7 +15,7 @@ func GetReleaseList(ctx context.Context, githubClient GitHubClient, organization } var values []models.Resource - opts := &github.ListOptions{PerPage: 100} + opts := &github.ListOptions{PerPage: releasePageSize} for _, r := range repositories { for { releases, resp, err := githubClient.RestClient.Repositories.ListReleases(ctx, organizationName, r.GetName(), opts) diff --git a/provider/describer/repository.go b/provider/describer/repository.go index 054b7aed..e0778a7c 100644 --- a/provider/describer/repository.go +++ b/provider/describer/repository.go @@ -144,6 +144,127 @@ func GetRepositoryList(ctx context.Context, githubClient GitHubClient, organizat return values, nil } +func GetRepository(ctx context.Context, githubClient GitHubClient, organizationName string, repositoryName string, resourceID string, stream *models.StreamSender) (*models.Resource, error) { + client := githubClient.GraphQLClient + query := struct { + RateLimit steampipemodels.RateLimit + Organization struct { + Repository steampipemodels.Repository `graphql:"repository(name: $repoName)"` + } `graphql:"organization(login: $owner)"` // <-- $owner used here + }{} + + variables := map[string]interface{}{ + "owner": githubv4.String(organizationName), + "repoName": githubv4.String(repositoryName), + } + + columnNames := repositoryCols() + appendRepoColumnIncludes(&variables, columnNames) + err := client.Query(ctx, &query, variables) + if err != nil { + return nil, err + } + repo := query.Organization.Repository + hooks, err := GetRepositoryHooks(ctx, githubClient.RestClient, organizationName, repo.Name) + if err != nil { + return nil, err + } + additionalRepoInfo, err := GetRepositoryAdditionalData(ctx, githubClient.RestClient, organizationName, repo.Name) + value := models.Resource{ + ID: strconv.Itoa(repo.Id), + Name: repo.Name, + Description: JSONAllFieldsMarshaller{ + Value: model.RepositoryDescription{ + ID: repo.Id, + NodeID: repo.NodeId, + Name: repo.Name, + AllowUpdateBranch: repo.AllowUpdateBranch, + ArchivedAt: repo.ArchivedAt, + AutoMergeAllowed: repo.AutoMergeAllowed, + CodeOfConduct: repo.CodeOfConduct, + ContactLinks: repo.ContactLinks, + CreatedAt: repo.CreatedAt, + DefaultBranchRef: repo.DefaultBranchRef, + DeleteBranchOnMerge: repo.DeleteBranchOnMerge, + Description: repo.Description, + DiskUsage: repo.DiskUsage, + ForkCount: repo.ForkCount, + ForkingAllowed: repo.ForkingAllowed, + FundingLinks: repo.FundingLinks, + HasDiscussionsEnabled: repo.HasDiscussionsEnabled, + HasIssuesEnabled: repo.HasIssuesEnabled, + HasProjectsEnabled: repo.HasProjectsEnabled, + HasVulnerabilityAlertsEnabled: repo.HasVulnerabilityAlertsEnabled, + HasWikiEnabled: repo.HasWikiEnabled, + HomepageURL: repo.HomepageUrl, + InteractionAbility: repo.InteractionAbility, + IsArchived: repo.IsArchived, + IsBlankIssuesEnabled: repo.IsBlankIssuesEnabled, + IsDisabled: repo.IsDisabled, + IsEmpty: repo.IsEmpty, + IsFork: repo.IsFork, + IsInOrganization: repo.IsInOrganization, + IsLocked: repo.IsLocked, + IsMirror: repo.IsMirror, + IsPrivate: repo.IsPrivate, + IsSecurityPolicyEnabled: repo.IsSecurityPolicyEnabled, + IsTemplate: repo.IsTemplate, + IsUserConfigurationRepository: repo.IsUserConfigurationRepository, + IssueTemplates: repo.IssueTemplates, + LicenseInfo: repo.LicenseInfo, + LockReason: repo.LockReason, + MergeCommitAllowed: repo.MergeCommitAllowed, + MergeCommitMessage: repo.MergeCommitMessage, + MergeCommitTitle: repo.MergeCommitTitle, + MirrorURL: repo.MirrorUrl, + NameWithOwner: repo.NameWithOwner, + OpenGraphImageURL: repo.OpenGraphImageUrl, + OwnerLogin: repo.Owner.Login, + PrimaryLanguage: repo.PrimaryLanguage, + ProjectsURL: repo.ProjectsUrl, + PullRequestTemplates: repo.PullRequestTemplates, + PushedAt: repo.PushedAt, + RebaseMergeAllowed: repo.RebaseMergeAllowed, + SecurityPolicyURL: repo.SecurityPolicyUrl, + SquashMergeAllowed: repo.SquashMergeAllowed, + SquashMergeCommitMessage: repo.SquashMergeCommitMessage, + SquashMergeCommitTitle: repo.SquashMergeCommitTitle, + SSHURL: repo.SshUrl, + StargazerCount: repo.StargazerCount, + UpdatedAt: repo.UpdatedAt, + URL: repo.Url, + UsesCustomOpenGraphImage: repo.UsesCustomOpenGraphImage, + CanAdminister: repo.CanAdminister, + CanCreateProjects: repo.CanCreateProjects, + CanSubscribe: repo.CanSubscribe, + CanUpdateTopics: repo.CanUpdateTopics, + HasStarred: repo.HasStarred, + PossibleCommitEmails: repo.PossibleCommitEmails, + Subscription: repo.Subscription, + Visibility: repo.Visibility, + YourPermission: repo.YourPermission, + WebCommitSignOffRequired: repo.WebCommitSignoffRequired, + RepositoryTopicsTotalCount: repo.RepositoryTopics.TotalCount, + OpenIssuesTotalCount: repo.OpenIssues.TotalCount, + WatchersTotalCount: repo.Watchers.TotalCount, + Hooks: hooks, + Topics: additionalRepoInfo.Topics, + SubscribersCount: additionalRepoInfo.GetSubscribersCount(), + HasDownloads: additionalRepoInfo.GetHasDownloads(), + HasPages: additionalRepoInfo.GetHasPages(), + NetworkCount: additionalRepoInfo.GetNetworkCount(), + }, + }, + } + if stream != nil { + if err := (*stream)(value); err != nil { + return nil, err + } + } + + return &value, nil +} + func GetRepositoryAdditionalData(ctx context.Context, client *github.Client, organizationName string, repo string) (*github.Repository, error) { repository, _, err := client.Repositories.Get(ctx, organizationName, repo) if err != nil { diff --git a/provider/describer/utils.go b/provider/describer/utils.go index a2509429..85c1fde3 100644 --- a/provider/describer/utils.go +++ b/provider/describer/utils.go @@ -18,11 +18,12 @@ type GitHubClient struct { const ( maxPagesCount = 100 pageSize = 100 - pullRequestsPageSize = 75 - repoPageSize = 50 + pullRequestsPageSize = 25 + repoPageSize = 15 issuePageSize = 50 orgPageSize = 10 orgCollaboratorsPageSize = 50 + releasePageSize = 100 teamMembersPageSize = 50 ) diff --git a/provider/resource_types.go b/provider/resource_types.go index 8d6905a1..52c547c8 100644 --- a/provider/resource_types.go +++ b/provider/resource_types.go @@ -297,7 +297,7 @@ var ResourceTypes = map[string]model.ResourceType{ Annotations: map[string]string{ }, ListDescriber: DescribeByGithub(describer.GetRepositoryList), - GetDescriber: nil, + GetDescriber: DescribeSingleByRepo(describer.GetRepository), }, "Github/Repository/Collaborator": { diff --git a/provider/resource_types/resource-types.json b/provider/resource_types/resource-types.json index 5f21ed5f..2b535bd7 100644 --- a/provider/resource_types/resource-types.json +++ b/provider/resource_types/resource-types.json @@ -247,7 +247,7 @@ ] }, "ListDescriber": "DescribeByGithub(describer.GetRepositoryList)", - "GetDescriber": "", + "GetDescriber": "DescribeSingleByRepo(describer.GetRepository)", "SteampipeTable": "github_repository", "Model": "Repository" },