From 7a373ff13befc748f9fa736d49640e71fd936e41 Mon Sep 17 00:00:00 2001 From: Patrick DeVivo Date: Wed, 29 Jun 2022 11:14:50 -0400 Subject: [PATCH 1/2] feat: implement a `github_org_audit_log` table Builds on previous work done by @Vialeon, adds basic support for querying the GitHub audit log via API (only available for enterprise plans) --- extensions/internal/github/github.go | 2 + extensions/internal/github/org_audit_log.go | 204 ++++++++++++++++++++ 2 files changed, 206 insertions(+) create mode 100644 extensions/internal/github/org_audit_log.go diff --git a/extensions/internal/github/github.go b/extensions/internal/github/github.go index f3f62292..e15c98b4 100644 --- a/extensions/internal/github/github.go +++ b/extensions/internal/github/github.go @@ -79,6 +79,7 @@ func Register(ext *sqlite.ExtensionApi, opt *options.Options) (_ sqlite.ErrorCod "github_repo_pr_commits": NewPRCommitsModule(githubOpts), "github_repo_commits": NewRepoCommitsModule(githubOpts), "github_repo_pr_reviews": NewPRReviewsModule(githubOpts), + "github_org_audit_log": NewOrgAuditModule(githubOpts), } modules["github_issue_comments"] = modules["github_repo_issue_comments"] @@ -90,6 +91,7 @@ func Register(ext *sqlite.ExtensionApi, opt *options.Options) (_ sqlite.ErrorCod modules["github_branch_protections"] = modules["github_repo_branch_protections"] modules["github_pr_commits"] = modules["github_repo_pr_commits"] modules["github_pr_reviews"] = modules["github_repo_pr_reviews"] + modules["github_audit_log"] = modules["github_org_audit_log"] // register GitHub tables for name, mod := range modules { diff --git a/extensions/internal/github/org_audit_log.go b/extensions/internal/github/org_audit_log.go new file mode 100644 index 00000000..b93ace6f --- /dev/null +++ b/extensions/internal/github/org_audit_log.go @@ -0,0 +1,204 @@ +package github + +import ( + "context" + "io" + "strings" + "time" + + "github.com/augmentable-dev/vtab" + "github.com/rs/zerolog" + "github.com/shurcooL/githubv4" + "go.riyazali.net/sqlite" +) + +type fetchOrgAuditLogResults struct { + AuditLogs []*auditLogEntry + HasNextPage bool + EndCursor *githubv4.String +} + +type auditLogEntry struct { + Typename string `graphql:"__typename"` + NodeFragment struct { + Id string + } `graphql:"... on Node"` + Entry auditLogEntryContents `graphql:"... on AuditEntry"` +} + +type auditLogEntryContents struct { + Action string + Actor struct { + Type string `graphql:"__typename"` + } + ActorLogin string + ActorIp string + CreatedAt githubv4.DateTime + OperationType string + UserLogin string +} + +func (i *iterOrgAuditLogs) fetchOrgAuditRepos(ctx context.Context, startCursor *githubv4.String) (*fetchOrgAuditLogResults, error) { + var reposQuery struct { + Organization struct { + Login string + AuditLog struct { + TotalCount int + Nodes []*auditLogEntry + PageInfo struct { + EndCursor githubv4.String + HasNextPage bool + } + } `graphql:"auditLog(first: $perPage, after: $auditLogCursor, orderBy: $auditLogOrder)"` + } `graphql:"organization(login: $login)"` + } + variables := map[string]interface{}{ + "login": githubv4.String(i.login), + "perPage": githubv4.Int(i.PerPage), + "auditLogCursor": startCursor, + "auditLogOrder": i.auditOrder, + } + + err := i.Client().Query(ctx, &reposQuery, variables) + if err != nil { + return nil, err + } + + return &fetchOrgAuditLogResults{ + reposQuery.Organization.AuditLog.Nodes, + reposQuery.Organization.AuditLog.PageInfo.HasNextPage, + &reposQuery.Organization.AuditLog.PageInfo.EndCursor, + }, nil + +} + +type iterOrgAuditLogs struct { + *Options + login string + affiliations string + current int + results *fetchOrgAuditLogResults + auditOrder *githubv4.AuditLogOrder +} + +func (i *iterOrgAuditLogs) logger() *zerolog.Logger { + logger := i.Logger.With().Int("per-page", i.PerPage).Str("login", i.login).Logger() + if i.auditOrder != nil { + logger = logger.With().Str("order_by", string(*i.auditOrder.Field)).Str("order_dir", string(*i.auditOrder.Direction)).Logger() + } + return &logger +} + +func (i *iterOrgAuditLogs) Column(ctx vtab.Context, c int) error { + current := i.results.AuditLogs[i.current] + + switch orgAuditCols[c].Name { + case "login": + ctx.ResultText(i.login) + case "id": + ctx.ResultText(current.NodeFragment.Id) + case "entry_type": + ctx.ResultText(current.Typename) + case "action": + ctx.ResultText(current.Entry.Action) + case "actor_type": + ctx.ResultText(current.Entry.Actor.Type) + case "actor_login": + ctx.ResultText(current.Entry.ActorLogin) + case "actor_ip": + ctx.ResultText(current.Entry.ActorIp) + case "created_at": + t := current.Entry.CreatedAt + if t.IsZero() { + ctx.ResultNull() + } else { + ctx.ResultText(t.Format(time.RFC3339Nano)) + } + case "operation_type": + ctx.ResultText(current.Entry.OperationType) + case "user_login": + ctx.ResultText(current.Entry.UserLogin) + } + return nil +} + +func (i *iterOrgAuditLogs) Next() (vtab.Row, error) { + i.current += 1 + + if i.results == nil || i.current >= len(i.results.AuditLogs) { + if i.results == nil || i.results.HasNextPage { + err := i.RateLimiter.Wait(context.Background()) + if err != nil { + return nil, err + } + + var cursor *githubv4.String + if i.results != nil { + cursor = i.results.EndCursor + } + + l := i.logger().With().Interface("cursor", cursor).Logger() + l.Info().Msgf("fetching page of org audit entries for %s", i.login) + results, err := i.fetchOrgAuditRepos(context.Background(), cursor) + if err != nil { + return nil, err + } + + i.results = results + i.current = 0 + + } else { + return nil, io.EOF + } + } + + return i, nil +} + +var orgAuditCols = []vtab.Column{ + {Name: "login", Type: "TEXT", Hidden: true, Filters: []*vtab.ColumnFilter{{Op: sqlite.INDEX_CONSTRAINT_EQ, OmitCheck: true}}}, + {Name: "id", Type: "TEXT"}, + {Name: "entry_type", Type: "TEXT"}, + {Name: "action", Type: "TEXT"}, + {Name: "actor_type", Type: "TEXT"}, + {Name: "actor_login", Type: "TEXT"}, + {Name: "actor_ip", Type: "TEXT"}, + {Name: "created_at", Type: "DATETIME", OrderBy: vtab.ASC | vtab.DESC}, + {Name: "operation_type", Type: "TEXT"}, + {Name: "user_login", Type: "TEXT"}, +} + +func NewOrgAuditModule(opts *Options) sqlite.Module { + return vtab.NewTableFunc("github_audit_repos", orgAuditCols, func(constraints []*vtab.Constraint, orders []*sqlite.OrderBy) (vtab.Iterator, error) { + var login, affiliations string + for _, constraint := range constraints { + if constraint.Op == sqlite.INDEX_CONSTRAINT_EQ { + switch constraint.ColIndex { + case 0: + login = constraint.Value.Text() + + case 1: + affiliations = strings.ToUpper(constraint.Value.Text()) + } + } + } + + var auditOrder *githubv4.AuditLogOrder + // for now we can only support single field order bys + if len(orders) == 1 { + order := orders[0] + switch orgAuditCols[order.ColumnIndex].Name { + case "created_at": + createdAt := githubv4.AuditLogOrderFieldCreatedAt + dir := orderByToGitHubOrder(order.Desc) + auditOrder = &githubv4.AuditLogOrder{ + Field: &createdAt, + } + auditOrder.Direction = &dir + } + } + iter := &iterOrgAuditLogs{opts, login, affiliations, -1, nil, auditOrder} + iter.logger().Info().Msgf("starting GitHub audit_log iterator for %s", login) + return iter, nil + }, vtab.EarlyOrderByConstraintExit(true)) +} From 67d4ea7db84bc23aab47e53ea6a685348ac9f917 Mon Sep 17 00:00:00 2001 From: Patrick DeVivo Date: Wed, 29 Jun 2022 11:26:35 -0400 Subject: [PATCH 2/2] feat: add a very simple test case (with anonymized data) --- .../github/fixtures/TestOrgAuditLog.yaml | 63 +++++++++++++++++++ .../internal/github/org_audit_log_test.go | 29 +++++++++ 2 files changed, 92 insertions(+) create mode 100644 extensions/internal/github/fixtures/TestOrgAuditLog.yaml create mode 100644 extensions/internal/github/org_audit_log_test.go diff --git a/extensions/internal/github/fixtures/TestOrgAuditLog.yaml b/extensions/internal/github/fixtures/TestOrgAuditLog.yaml new file mode 100644 index 00000000..117ae900 --- /dev/null +++ b/extensions/internal/github/fixtures/TestOrgAuditLog.yaml @@ -0,0 +1,63 @@ +--- +version: 1 +interactions: +- request: + body: | + {"query":"query($auditLogCursor:String$auditLogOrder:AuditLogOrder$login:String!$perPage:Int!){organization(login: $login){login,auditLog(first: $perPage, after: $auditLogCursor, orderBy: $auditLogOrder){totalCount,nodes{__typename,... on Node{id},... on AuditEntry{action,actor{__typename},actorLogin,actorIp,createdAt,operationType,userLogin}},pageInfo{endCursor,hasNextPage}}}}","variables":{"auditLogCursor":null,"auditLogOrder":null,"login":"mergestat","perPage":50}} + form: {} + headers: + Content-Type: + - application/json + url: https://api.github.com/graphql + method: POST + response: + body: '{"data":{"organization":{"login":"mergestat","auditLog":{"totalCount":39,"nodes":[{"__typename":"RepoCreateAuditEntry","id":"fake-id","action":"repo.create","actor":{"__typename":"User"},"actorLogin":"patrickdevivo","actorIp":"0.0.0.0","createdAt":"2022-06-29T14:06:02.543Z","operationType":"CREATE","userLogin":null}],"pageInfo":{"endCursor":null,"hasNextPage":false}}}}}' + headers: + Access-Control-Allow-Origin: + - '*' + Access-Control-Expose-Headers: + - ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, + X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, + X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, + X-GitHub-Request-Id, Deprecation, Sunset + Content-Security-Policy: + - default-src 'none' + Content-Type: + - application/json; charset=utf-8 + Date: + - Wed, 29 Jun 2022 15:22:01 GMT + Referrer-Policy: + - origin-when-cross-origin, strict-origin-when-cross-origin + Server: + - GitHub.com + Strict-Transport-Security: + - max-age=31536000; includeSubdomains; preload + Vary: + - Accept-Encoding, Accept, X-Requested-With + X-Accepted-Oauth-Scopes: + - repo + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - deny + X-Github-Media-Type: + - github.v4; format=json + X-Github-Request-Id: + - D50C:5249:26D5CB9:574FAA0:62BC6E18 + X-Oauth-Scopes: + - admin:enterprise, admin:org, project, repo, user + X-Ratelimit-Limit: + - "5000" + X-Ratelimit-Remaining: + - "4970" + X-Ratelimit-Reset: + - "1656516380" + X-Ratelimit-Resource: + - graphql + X-Ratelimit-Used: + - "30" + X-Xss-Protection: + - "0" + status: 200 OK + code: 200 + duration: 1.438554716s diff --git a/extensions/internal/github/org_audit_log_test.go b/extensions/internal/github/org_audit_log_test.go new file mode 100644 index 00000000..1e364a57 --- /dev/null +++ b/extensions/internal/github/org_audit_log_test.go @@ -0,0 +1,29 @@ +package github_test + +import ( + "testing" + + "github.com/mergestat/mergestat/extensions/internal/tools" +) + +func TestOrgAuditLog(t *testing.T) { + cleanup := newRecorder(t) + defer cleanup() + + db := Connect(t, Memory) + + rows, err := db.Query("SELECT * FROM github_org_audit_log('mergestat') LIMIT 1") + if err != nil { + t.Fatalf("failed to execute query: %v", err.Error()) + } + defer rows.Close() + + colCount, _, err := tools.RowContent(rows) + if err != nil { + t.Fatalf("failed to retrieve row contents: %v", err.Error()) + } + + if expected := 9; colCount != expected { + t.Fatalf("expected %d columns, got: %d", expected, colCount) + } +}