From 0ee7be967d78b73aa21830268671e568ac86e70d Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Tue, 27 Jul 2021 18:26:28 -0400 Subject: [PATCH 1/2] create binary diffs in parallel --- cmd/go-selfupdate/main.go | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/cmd/go-selfupdate/main.go b/cmd/go-selfupdate/main.go index 08e33f7..62170dd 100644 --- a/cmd/go-selfupdate/main.go +++ b/cmd/go-selfupdate/main.go @@ -8,10 +8,12 @@ import ( "flag" "fmt" "io" + "io/fs" "io/ioutil" "os" "path/filepath" "runtime" + "sync" "github.com/kr/binarydist" ) @@ -83,17 +85,12 @@ func createUpdate(path string, platform string) { w.Close() // You must close this first to flush the bytes to the buffer. err = ioutil.WriteFile(filepath.Join(genDir, version, platform+".gz"), buf.Bytes(), 0755) - files, err := ioutil.ReadDir(genDir) - if err != nil { - fmt.Println(err) - } - - for _, file := range files { + processUpdate := func(file fs.FileInfo) { if file.IsDir() == false { - continue + return } if file.Name() == version { - continue + return } os.Mkdir(filepath.Join(genDir, file.Name(), version), 0755) @@ -102,7 +99,7 @@ func createUpdate(path string, platform string) { old, err := os.Open(fName) if err != nil { // Don't have an old release for this os/arch, continue on - continue + return } fName = filepath.Join(genDir, version, platform+".gz") @@ -122,6 +119,30 @@ func createUpdate(path string, platform string) { } ioutil.WriteFile(filepath.Join(genDir, file.Name(), version, platform), patch.Bytes(), 0755) } + + files, err := ioutil.ReadDir(genDir) + if err != nil { + fmt.Println(err) + } + + // spin up parallel workers to process the files: + numWorkers := runtime.NumCPU() + filesChan := make(chan fs.FileInfo) + var wg sync.WaitGroup + wg.Add(numWorkers) + for i := 0; i < numWorkers; i++ { + go func() { + for file := range filesChan { + processUpdate(file) + } + wg.Done() + }() + } + for _, file := range files { + filesChan <- file + } + close(filesChan) + wg.Wait() } func printUsage() { From 48692d640e22e5309e2f43d35e34e466666781b1 Mon Sep 17 00:00:00 2001 From: Shivaram Lingamneni Date: Tue, 27 Jul 2021 18:33:11 -0400 Subject: [PATCH 2/2] don't update the metadata file until success Wait for all updates to be successfully processed before updating the file containing the latest version number. --- cmd/go-selfupdate/main.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/cmd/go-selfupdate/main.go b/cmd/go-selfupdate/main.go index 62170dd..0492725 100644 --- a/cmd/go-selfupdate/main.go +++ b/cmd/go-selfupdate/main.go @@ -62,17 +62,6 @@ func newGzReader(r io.ReadCloser) io.ReadCloser { } func createUpdate(path string, platform string) { - c := current{Version: version, Sha256: generateSha256(path)} - - b, err := json.MarshalIndent(c, "", " ") - if err != nil { - fmt.Println("error:", err) - } - err = ioutil.WriteFile(filepath.Join(genDir, platform+".json"), b, 0755) - if err != nil { - panic(err) - } - os.MkdirAll(filepath.Join(genDir, version), 0755) var buf bytes.Buffer @@ -105,8 +94,7 @@ func createUpdate(path string, platform string) { fName = filepath.Join(genDir, version, platform+".gz") newF, err := os.Open(fName) if err != nil { - fmt.Fprintf(os.Stderr, "Can't open %s: error: %s\n", fName, err) - os.Exit(1) + panic(fmt.Sprintf("Can't open %s: error: %s", fName, err)) } ar := newGzReader(old) @@ -143,6 +131,16 @@ func createUpdate(path string, platform string) { } close(filesChan) wg.Wait() + + c := current{Version: version, Sha256: generateSha256(path)} + b, err := json.MarshalIndent(c, "", " ") + if err != nil { + fmt.Println("error:", err) + } + err = ioutil.WriteFile(filepath.Join(genDir, platform+".json"), b, 0755) + if err != nil { + panic(err) + } } func printUsage() {