Merge pull request #295 from mergestat/github-audit-log

feat: implement a `github_org_audit_log` table

extensions/internal/github/fixtures/TestOrgAuditLog.yaml

@@ -0,0 +1,63 @@
+---
+version: 1
+interactions:
+- request:
+    body: |
+      {"query":"query($auditLogCursor:String$auditLogOrder:AuditLogOrder$login:String!$perPage:Int!){organization(login: $login){login,auditLog(first: $perPage, after: $auditLogCursor, orderBy: $auditLogOrder){totalCount,nodes{__typename,... on Node{id},... on AuditEntry{action,actor{__typename},actorLogin,actorIp,createdAt,operationType,userLogin}},pageInfo{endCursor,hasNextPage}}}}","variables":{"auditLogCursor":null,"auditLogOrder":null,"login":"mergestat","perPage":50}}
+    form: {}
+    headers:
+      Content-Type:
+      - application/json
+    url: https://api.github.com/graphql
+    method: POST
+  response:
+    body: '{"data":{"organization":{"login":"mergestat","auditLog":{"totalCount":39,"nodes":[{"__typename":"RepoCreateAuditEntry","id":"fake-id","action":"repo.create","actor":{"__typename":"User"},"actorLogin":"patrickdevivo","actorIp":"0.0.0.0","createdAt":"2022-06-29T14:06:02.543Z","operationType":"CREATE","userLogin":null}],"pageInfo":{"endCursor":null,"hasNextPage":false}}}}}'
+    headers:
+      Access-Control-Allow-Origin:
+      - '*'
+      Access-Control-Expose-Headers:
+      - ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining,
+        X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes,
+        X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO,
+        X-GitHub-Request-Id, Deprecation, Sunset
+      Content-Security-Policy:
+      - default-src 'none'
+      Content-Type:
+      - application/json; charset=utf-8
+      Date:
+      - Wed, 29 Jun 2022 15:22:01 GMT
+      Referrer-Policy:
+      - origin-when-cross-origin, strict-origin-when-cross-origin
+      Server:
+      - GitHub.com
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubdomains; preload
+      Vary:
+      - Accept-Encoding, Accept, X-Requested-With
+      X-Accepted-Oauth-Scopes:
+      - repo
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-Github-Media-Type:
+      - github.v4; format=json
+      X-Github-Request-Id:
+      - D50C:5249:26D5CB9:574FAA0:62BC6E18
+      X-Oauth-Scopes:
+      - admin:enterprise, admin:org, project, repo, user
+      X-Ratelimit-Limit:
+      - "5000"
+      X-Ratelimit-Remaining:
+      - "4970"
+      X-Ratelimit-Reset:
+      - "1656516380"
+      X-Ratelimit-Resource:
+      - graphql
+      X-Ratelimit-Used:
+      - "30"
+      X-Xss-Protection:
+      - "0"
+    status: 200 OK
+    code: 200
+    duration: 1.438554716s

extensions/internal/github/github.go

@@ -79,6 +79,7 @@ func Register(ext *sqlite.ExtensionApi, opt *options.Options) (_ sqlite.ErrorCod
 		"github_repo_pr_commits":         NewPRCommitsModule(githubOpts),
 		"github_repo_commits":            NewRepoCommitsModule(githubOpts),
 		"github_repo_pr_reviews":         NewPRReviewsModule(githubOpts),
+		"github_org_audit_log":           NewOrgAuditModule(githubOpts),
 	}
 
 	modules["github_issue_comments"] = modules["github_repo_issue_comments"]
@@ -90,6 +91,7 @@ func Register(ext *sqlite.ExtensionApi, opt *options.Options) (_ sqlite.ErrorCod
 	modules["github_branch_protections"] = modules["github_repo_branch_protections"]
 	modules["github_pr_commits"] = modules["github_repo_pr_commits"]
 	modules["github_pr_reviews"] = modules["github_repo_pr_reviews"]
+	modules["github_audit_log"] = modules["github_org_audit_log"]
 
 	// register GitHub tables
 	for name, mod := range modules {

extensions/internal/github/org_audit_log.go

@@ -0,0 +1,204 @@
+package github
+
+import (
+	"context"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/augmentable-dev/vtab"
+	"github.com/rs/zerolog"
+	"github.com/shurcooL/githubv4"
+	"go.riyazali.net/sqlite"
+)
+
+type fetchOrgAuditLogResults struct {
+	AuditLogs   []*auditLogEntry
+	HasNextPage bool
+	EndCursor   *githubv4.String
+}
+
+type auditLogEntry struct {
+	Typename     string `graphql:"__typename"`
+	NodeFragment struct {
+		Id string
+	} `graphql:"... on Node"`
+	Entry auditLogEntryContents `graphql:"... on AuditEntry"`
+}
+
+type auditLogEntryContents struct {
+	Action string
+	Actor  struct {
+		Type string `graphql:"__typename"`
+	}
+	ActorLogin    string
+	ActorIp       string
+	CreatedAt     githubv4.DateTime
+	OperationType string
+	UserLogin     string
+}
+
+func (i *iterOrgAuditLogs) fetchOrgAuditRepos(ctx context.Context, startCursor *githubv4.String) (*fetchOrgAuditLogResults, error) {
+	var reposQuery struct {
+		Organization struct {
+			Login    string
+			AuditLog struct {
+				TotalCount int
+				Nodes      []*auditLogEntry
+				PageInfo   struct {
+					EndCursor   githubv4.String
+					HasNextPage bool
+				}
+			} `graphql:"auditLog(first: $perPage, after: $auditLogCursor, orderBy: $auditLogOrder)"`
+		} `graphql:"organization(login: $login)"`
+	}
+	variables := map[string]interface{}{
+		"login":          githubv4.String(i.login),
+		"perPage":        githubv4.Int(i.PerPage),
+		"auditLogCursor": startCursor,
+		"auditLogOrder":  i.auditOrder,
+	}
+
+	err := i.Client().Query(ctx, &reposQuery, variables)
+	if err != nil {
+		return nil, err
+	}
+
+	return &fetchOrgAuditLogResults{
+		reposQuery.Organization.AuditLog.Nodes,
+		reposQuery.Organization.AuditLog.PageInfo.HasNextPage,
+		&reposQuery.Organization.AuditLog.PageInfo.EndCursor,
+	}, nil
+
+}
+
+type iterOrgAuditLogs struct {
+	*Options
+	login        string
+	affiliations string
+	current      int
+	results      *fetchOrgAuditLogResults
+	auditOrder   *githubv4.AuditLogOrder
+}
+
+func (i *iterOrgAuditLogs) logger() *zerolog.Logger {
+	logger := i.Logger.With().Int("per-page", i.PerPage).Str("login", i.login).Logger()
+	if i.auditOrder != nil {
+		logger = logger.With().Str("order_by", string(*i.auditOrder.Field)).Str("order_dir", string(*i.auditOrder.Direction)).Logger()
+	}
+	return &logger
+}
+
+func (i *iterOrgAuditLogs) Column(ctx vtab.Context, c int) error {
+	current := i.results.AuditLogs[i.current]
+
+	switch orgAuditCols[c].Name {
+	case "login":
+		ctx.ResultText(i.login)
+	case "id":
+		ctx.ResultText(current.NodeFragment.Id)
+	case "entry_type":
+		ctx.ResultText(current.Typename)
+	case "action":
+		ctx.ResultText(current.Entry.Action)
+	case "actor_type":
+		ctx.ResultText(current.Entry.Actor.Type)
+	case "actor_login":
+		ctx.ResultText(current.Entry.ActorLogin)
+	case "actor_ip":
+		ctx.ResultText(current.Entry.ActorIp)
+	case "created_at":
+		t := current.Entry.CreatedAt
+		if t.IsZero() {
+			ctx.ResultNull()
+		} else {
+			ctx.ResultText(t.Format(time.RFC3339Nano))
+		}
+	case "operation_type":
+		ctx.ResultText(current.Entry.OperationType)
+	case "user_login":
+		ctx.ResultText(current.Entry.UserLogin)
+	}
+	return nil
+}
+
+func (i *iterOrgAuditLogs) Next() (vtab.Row, error) {
+	i.current += 1
+
+	if i.results == nil || i.current >= len(i.results.AuditLogs) {
+		if i.results == nil || i.results.HasNextPage {
+			err := i.RateLimiter.Wait(context.Background())
+			if err != nil {
+				return nil, err
+			}
+
+			var cursor *githubv4.String
+			if i.results != nil {
+				cursor = i.results.EndCursor
+			}
+
+			l := i.logger().With().Interface("cursor", cursor).Logger()
+			l.Info().Msgf("fetching page of org audit entries for %s", i.login)
+			results, err := i.fetchOrgAuditRepos(context.Background(), cursor)
+			if err != nil {
+				return nil, err
+			}
+
+			i.results = results
+			i.current = 0
+
+		} else {
+			return nil, io.EOF
+		}
+	}
+
+	return i, nil
+}
+
+var orgAuditCols = []vtab.Column{
+	{Name: "login", Type: "TEXT", Hidden: true, Filters: []*vtab.ColumnFilter{{Op: sqlite.INDEX_CONSTRAINT_EQ, OmitCheck: true}}},
+	{Name: "id", Type: "TEXT"},
+	{Name: "entry_type", Type: "TEXT"},
+	{Name: "action", Type: "TEXT"},
+	{Name: "actor_type", Type: "TEXT"},
+	{Name: "actor_login", Type: "TEXT"},
+	{Name: "actor_ip", Type: "TEXT"},
+	{Name: "created_at", Type: "DATETIME", OrderBy: vtab.ASC | vtab.DESC},
+	{Name: "operation_type", Type: "TEXT"},
+	{Name: "user_login", Type: "TEXT"},
+}
+
+func NewOrgAuditModule(opts *Options) sqlite.Module {
+	return vtab.NewTableFunc("github_audit_repos", orgAuditCols, func(constraints []*vtab.Constraint, orders []*sqlite.OrderBy) (vtab.Iterator, error) {
+		var login, affiliations string
+		for _, constraint := range constraints {
+			if constraint.Op == sqlite.INDEX_CONSTRAINT_EQ {
+				switch constraint.ColIndex {
+				case 0:
+					login = constraint.Value.Text()
+
+				case 1:
+					affiliations = strings.ToUpper(constraint.Value.Text())
+				}
+			}
+		}
+
+		var auditOrder *githubv4.AuditLogOrder
+		// for now we can only support single field order bys
+		if len(orders) == 1 {
+			order := orders[0]
+			switch orgAuditCols[order.ColumnIndex].Name {
+			case "created_at":
+				createdAt := githubv4.AuditLogOrderFieldCreatedAt
+				dir := orderByToGitHubOrder(order.Desc)
+				auditOrder = &githubv4.AuditLogOrder{
+					Field: &createdAt,
+				}
+				auditOrder.Direction = &dir
+			}
+		}
+		iter := &iterOrgAuditLogs{opts, login, affiliations, -1, nil, auditOrder}
+		iter.logger().Info().Msgf("starting GitHub audit_log iterator for %s", login)
+		return iter, nil
+	}, vtab.EarlyOrderByConstraintExit(true))
+}

extensions/internal/github/org_audit_log_test.go

@@ -0,0 +1,29 @@
+package github_test
+
+import (
+	"testing"
+
+	"github.com/mergestat/mergestat/extensions/internal/tools"
+)
+
+func TestOrgAuditLog(t *testing.T) {
+	cleanup := newRecorder(t)
+	defer cleanup()
+
+	db := Connect(t, Memory)
+
+	rows, err := db.Query("SELECT * FROM github_org_audit_log('mergestat') LIMIT 1")
+	if err != nil {
+		t.Fatalf("failed to execute query: %v", err.Error())
+	}
+	defer rows.Close()
+
+	colCount, _, err := tools.RowContent(rows)
+	if err != nil {
+		t.Fatalf("failed to retrieve row contents: %v", err.Error())
+	}
+
+	if expected := 9; colCount != expected {
+		t.Fatalf("expected %d columns, got: %d", expected, colCount)
+	}
+}