diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 57d4390..c08537b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -47,7 +47,7 @@ jobs: - name: Run tests with coverage run: | - go test ./... --race --covermode=atomic --coverprofile=coverage.out + go test ./... -v --race --covermode=atomic --coverprofile=coverage.out - name: Upload coverage reports uses: codecov/codecov-action@v3 diff --git a/catalyst-uploader.go b/catalyst-uploader.go index 6646d65..465b3fb 100644 --- a/catalyst-uploader.go +++ b/catalyst-uploader.go @@ -3,10 +3,10 @@ package main import ( "encoding/json" "flag" - "log" "os" "time" + "github.com/golang/glog" "github.com/livepeer/catalyst-uploader/core" "github.com/livepeer/go-tools/drivers" ) @@ -32,7 +32,7 @@ func run() int { } if flag.NArg() == 0 { - log.Fatal("Destination URI is not specified. See -h for usage.") + glog.Fatal("Destination URI is not specified. See -h for usage.") return 1 } @@ -42,20 +42,20 @@ func run() int { uri := flag.Arg(0) if uri == "" { - log.Fatalf("Could not parse object store URI: %s", uri) + glog.Fatalf("Could not parse object store URI: %s", uri) return 1 } err := core.Upload(os.Stdin, uri, WaitBetweenWrites, *timeout) if err != nil { - log.Fatalf("Uploader failed for %s: %s", uri, err) + glog.Fatalf("Uploader failed for %s: %s", uri, err) return 1 } // success, write uploaded file details to stdout err = json.NewEncoder(stdout).Encode(map[string]string{"uri": uri}) if err != nil { - log.Println(err) + glog.Fatal(err) return 1 } diff --git a/core/uploader.go b/core/uploader.go index 8ee5ec5..b835094 100644 --- a/core/uploader.go +++ b/core/uploader.go @@ -6,7 +6,6 @@ import ( "context" "fmt" "io" - "log" "os" "os/exec" "path/filepath" @@ -14,6 +13,7 @@ import ( "time" "github.com/cenkalti/backoff/v4" + "github.com/golang/glog" "github.com/livepeer/go-tools/drivers" ) @@ -62,7 +62,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t err = backoff.Retry(func() error { _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout) if err != nil { - log.Printf("failed upload attempt: %s", err) + glog.Errorf("failed upload attempt: %v", err) } return err }, UploadRetryBackoff()) @@ -71,7 +71,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t } if err = extractThumb(session, outputURI, fileContents); err != nil { - log.Printf("extracting thumbnail failed: %s", err) + glog.Errorf("extracting thumbnail failed: %v", err) } return nil } @@ -99,16 +99,16 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t b := scanner.Bytes() fileContents = append(fileContents, b...) if strings.Contains(outputURI, "m3u8") { - log.Printf("Received new bytes for %s: %s", outputURI, string(b)) + glog.V(5).Infof("Received new bytes for %s: %s", outputURI, string(b)) } // Only write the latest version of the data that's been piped in if enough time has elapsed since the last write if lastWrite.Add(waitBetweenWrites).Before(time.Now()) { if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout); err != nil { // Just log this error, since it'll effectively be retried after the next interval - log.Printf("Failed to write: %s", err) + glog.Errorf("Failed to write: %v", err) } else { - log.Printf("Wrote %s to storage: %d bytes", outputURI, len(b)) + glog.V(5).Infof("Wrote %s to storage: %d bytes", outputURI, len(b)) } lastWrite = time.Now() } @@ -122,7 +122,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t // Don't ignore this error, since there won't be any further attempts to write return fmt.Errorf("failed to write final save: %w", err) } - + glog.Infof("Completed writing %s to storage", outputURI) return nil } diff --git a/go.mod b/go.mod index 71c2b8f..eadf0d5 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/livepeer/catalyst-uploader go 1.17 require ( + github.com/cenkalti/backoff/v4 v4.2.1 + github.com/golang/glog v1.1.0 github.com/google/uuid v1.3.0 github.com/livepeer/go-tools v0.3.3 github.com/stretchr/testify v1.8.4 @@ -15,7 +17,6 @@ require ( cloud.google.com/go/iam v1.1.0 // indirect cloud.google.com/go/storage v1.30.1 // indirect github.com/aws/aws-sdk-go v1.44.273 // indirect - github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 54a8efd..16547a4 100644 --- a/go.sum +++ b/go.sum @@ -852,6 +852,7 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=