From 35ed68d0e4883a8240a7d026650c415d80613e4d Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Mon, 18 Dec 2023 22:03:43 +0100 Subject: [PATCH] loop is working with cursor rest cursor from github is not working as intended so there is some ineffective code to grab all the call --- gosmee/flags.go | 32 ++++++++- gosmee/replay.go | 175 +++++++++++++++++++++++++++++++++-------------- 2 files changed, 155 insertions(+), 52 deletions(-) diff --git a/gosmee/flags.go b/gosmee/flags.go index 52e2c5e..466a774 100644 --- a/gosmee/flags.go +++ b/gosmee/flags.go @@ -1,6 +1,31 @@ package gosmee -import "github.com/urfave/cli/v2" +import ( + "os" + + "github.com/urfave/cli/v2" +) + +func getCachePath() string { + cachePath := "/tmp/gosmee" + if os.Getenv("GOSMEE_CACHE_PATH") != "" { + cachePath = os.Getenv("GOSMEE_CACHE_PATH") + } else if os.Getenv("XDG_CACHE_HOME") != "" { + cachePath = os.Getenv("XDG_CACHE_HOME") + "/gosmee" + } else if os.Getenv("HOME") != "" { + cachePath = os.Getenv("HOME") + "/.cache/gosmee" + } + // create base dir if not exists + if _, err := os.Stat(cachePath); os.IsNotExist(err) { + err := os.MkdirAll(cachePath, 0o755) + if err != nil { + panic(err) + } + } + return cachePath +} + +var cachePath = getCachePath() var commonFlags = []cli.Flag{ &cli.StringSliceFlag{ @@ -50,6 +75,11 @@ var replayFlags = []cli.Flag{ Usage: "List hooks and its a IDs from a repository", Aliases: []string{"L"}, }, + &cli.StringFlag{ + Name: "cache-dir", + Usage: "Cache dir where to store the last delivered event", + Value: cachePath, + }, } var clientFlags = []cli.Flag{ diff --git a/gosmee/replay.go b/gosmee/replay.go index 365256f..6712f19 100644 --- a/gosmee/replay.go +++ b/gosmee/replay.go @@ -5,8 +5,10 @@ import ( "fmt" "net/url" "os" + "path/filepath" "strconv" "strings" + "time" "github.com/google/go-github/v57/github" "github.com/mattn/go-isatty" @@ -20,58 +22,127 @@ type replayOpts struct { client *github.Client repo string org string + cacheDir string +} + +func saveCursor(cacheFile, cursor string) error { + f, err := os.OpenFile(cacheFile, os.O_RDWR|os.O_CREATE, 0o600) + if err != nil { + return fmt.Errorf("cannot open cache file: %w", err) + } + defer f.Close() + _, err = f.WriteString(cursor) + if err != nil { + return fmt.Errorf("cannot write to cache file: %w", err) + } + return nil +} + +func readCursor(cacheFile string) (string, error) { + f, err := os.OpenFile(cacheFile, os.O_RDONLY, 0o600) + if err != nil { + return "", fmt.Errorf("cannot open cache file: %w", err) + } + defer f.Close() + buf := make([]byte, 1024) + n, err := f.Read(buf) + if err != nil { + return "", fmt.Errorf("cannot read cache file: %w", err) + } + return strings.TrimSpace(string(buf[:n])), nil +} + +// chooseDeliveries reverses the deliveries slice and only show the deliveries since the cache file id +func chooseDeliveries(deliveries []*github.HookDelivery, cacheFileID string) []*github.HookDelivery { + for i := len(deliveries)/2 - 1; i >= 0; i-- { + opp := len(deliveries) - 1 - i + deliveries[i], deliveries[opp] = deliveries[opp], deliveries[i] + } + if cacheFileID == "" { + return deliveries + } + iCacheFileID, err := strconv.ParseInt(cacheFileID, 10, 64) + if err != nil { + return deliveries + } + + retdeliveries := make([]*github.HookDelivery, 0) + for _, d := range deliveries { + if d.GetID() == iCacheFileID { + retdeliveries = []*github.HookDelivery{} + continue + } + retdeliveries = append(retdeliveries, d) + } + return retdeliveries } func (r *replayOpts) replayHooks(ctx context.Context, hookid int64) error { + cacheFile := filepath.Join(r.cacheDir, fmt.Sprintf("%s-%s-%d", r.org, r.repo, hookid)) opt := &github.ListCursorOptions{} + cursor, _ := readCursor(cacheFile) + var changed bool for { - deliveries, resp, err := r.client.Repositories.ListHookDeliveries(ctx, r.org, r.repo, hookid, opt) - if err != nil { - return fmt.Errorf("cannot list deliveries: %w", err) - } - - for _, hd := range deliveries { - delivery, _, err := r.client.Repositories.GetHookDelivery(ctx, r.org, r.repo, hookid, hd.GetID()) + for { + deliveries, resp, err := r.client.Repositories.ListHookDeliveries(ctx, r.org, r.repo, hookid, opt) if err != nil { - return fmt.Errorf("cannot get delivery: %w", err) - } - pm := payloadMsg{} - var ok bool - if pm.contentType, ok = delivery.Request.Headers["Content-Type"]; !ok { - pm.contentType = "application/json" + return fmt.Errorf("cannot list deliveries: %w", err) } - pm.body = delivery.Request.GetRawPayload() - pm.headers = delivery.Request.GetHeaders() - - // get the event type - if pv, ok := pm.headers["X-GitHub-Event"]; ok { - // github action don't like it - replace := strings.NewReplacer(":", "-", " ", "_", "/", "_") - pv = replace.Replace(strings.ToLower(pv)) - // remove all non-alphanumeric characters and don't let directory traversal - pv = pmEventRe.FindString(pv) - pm.eventType = pv + // reverse deliveries to replay from oldest to newest + deliveries = chooseDeliveries(deliveries, cursor) + for _, hd := range deliveries { + cursor = fmt.Sprintf("%d", hd.GetID()) + changed = true + delivery, _, err := r.client.Repositories.GetHookDelivery(ctx, r.org, r.repo, hookid, hd.GetID()) + if err != nil { + return fmt.Errorf("cannot get delivery: %w", err) + } + pm := payloadMsg{} + var ok bool + if pm.contentType, ok = delivery.Request.Headers["Content-Type"]; !ok { + pm.contentType = "application/json" + } + pm.body = delivery.Request.GetRawPayload() + pm.headers = delivery.Request.GetHeaders() + + // get the event type + if pv, ok := pm.headers["X-GitHub-Event"]; ok { + // github action don't like it + replace := strings.NewReplacer(":", "-", " ", "_", "/", "_") + pv = replace.Replace(strings.ToLower(pv)) + // remove all non-alphanumeric characters and don't let directory traversal + pv = pmEventRe.FindString(pv) + pm.eventType = pv + } + if pd, ok := pm.headers["X-GitHub-Delivery"]; ok { + pm.eventID = pd + } + + dt := delivery.DeliveredAt.GetTime() + pm.timestamp = dt.Format(tsFormat) + + if err := replayData(r.replayDataOpts, pm); err != nil { + fmt.Fprintf(os.Stdout, + "%s forwarding message with headers '%s' - %s\n", + ansi.Color("ERROR", "red+b"), + pm.headers, + err.Error()) + continue + } } - if pd, ok := pm.headers["X-GitHub-Delivery"]; ok { - pm.eventID = pd - } - dt := delivery.DeliveredAt.GetTime() - pm.timestamp = dt.Format(tsFormat) - - if err := replayData(r.replayDataOpts, pm); err != nil { - fmt.Fprintf(os.Stdout, - "%s forwarding message with headers '%s' - %s\n", - ansi.Color("ERROR", "red+b"), - pm.headers, - err.Error()) - continue + + if resp.NextPage == 0 { + break } } - - if resp.NextPage == 0 { - break + // save the cursor to cache file + if changed { + if err := saveCursor(cacheFile, cursor); err != nil { + fmt.Fprintf(os.Stdout, "error saving cursor to cache file %s: %s\n", r.cacheDir, err.Error()) + break + } } - opt.Cursor = resp.Cursor + time.Sleep(5 * time.Second) } return nil } @@ -98,22 +169,24 @@ func replay(c *cli.Context) error { client := github.NewClient(nil) client = client.WithAuthToken(c.String("github-token")) + ropt := &replayOpts{ + cliCtx: c, + client: client, + cacheDir: cachePath, + } + if c.String("cache-dir") != "" { + ropt.cacheDir = c.String("cache-dir") + } + orgRepo := c.Args().Get(0) - var org, repo string if strings.Contains(orgRepo, "/") { - org = strings.Split(orgRepo, "/")[1] - repo = strings.Split(orgRepo, "/")[0] + ropt.org = strings.Split(orgRepo, "/")[0] + ropt.repo = strings.Split(orgRepo, "/")[1] } - if org == "" || repo == "" { + if ropt.org == "" || ropt.repo == "" { return fmt.Errorf("org and repo are required, example: org/repo") } - ropt := &replayOpts{ - cliCtx: c, - client: client, - repo: org, - org: repo, - } if c.IsSet("list-hooks") { return ropt.listHooks(ctx) }