diff --git a/catalyst-uploader.go b/catalyst-uploader.go index d6cf3b0..9bfc65b 100644 --- a/catalyst-uploader.go +++ b/catalyst-uploader.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "strings" "time" "github.com/golang/glog" @@ -37,7 +38,7 @@ func run() int { describe := fs.Bool("j", false, "Describe supported storage services in JSON format and exit") verbosity := fs.String("v", "", "Log verbosity. {4|5|6}") timeout := fs.Duration("t", 30*time.Second, "Upload timeout") - storageBackupURLs := jsonFlag[core.StorageBackupURLs](fs, "storage-backup-urls", `JSON array of {"primary":X,"backup":Y} objects with base storage URLs. If a file fails uploading to one of the primary storages (detected by prefix), it will fallback to the corresponding backup URL after having the prefix replaced`) + storageBackupURLs := CommaMapFlag(fs, "storage-backup-urls", `Comma-separated map of primary to backup storage URLs. If a file fails uploading to one of the primary storages (detected by prefix), it will fallback to the corresponding backup URL after having the prefix replaced`) _ = fs.String("config", "", "config file (optional)") @@ -94,7 +95,7 @@ func run() int { return 1 } - out, err := core.Upload(os.Stdin, uri, WaitBetweenWrites, *timeout, storageBackupURLs) + out, err := core.Upload(os.Stdin, uri, WaitBetweenWrites, *timeout, *storageBackupURLs) if err != nil { glog.Errorf("Uploader failed for %s: %s", uri.Redacted(), err) return 1 @@ -117,10 +118,29 @@ func run() int { return 0 } -func jsonFlag[T any](fs *flag.FlagSet, name string, usage string) T { - var value T +// handles -foo=key1=value1,key2=value2 +func CommaMapFlag(fs *flag.FlagSet, name string, usage string) *map[string]string { + var dest map[string]string fs.Func(name, usage, func(s string) error { - return json.Unmarshal([]byte(s), &value) + var err error + dest, err = parseCommaMap(s) + return err }) - return value + return &dest +} + +func parseCommaMap(s string) (map[string]string, error) { + output := map[string]string{} + if s == "" { + return output, nil + } + for _, pair := range strings.Split(s, ",") { + kv := strings.Split(pair, "=") + if len(kv) != 2 { + return map[string]string{}, fmt.Errorf("failed to parse keypairs, -option=k1=v1,k2=v2 format required, got %s", s) + } + k, v := kv[0], kv[1] + output[k] = v + } + return output, nil } diff --git a/core/uploader.go b/core/uploader.go index f80cbf0..b148752 100644 --- a/core/uploader.go +++ b/core/uploader.go @@ -46,12 +46,7 @@ var expiryField = map[string]string{ "Object-Expires": "+168h", // Objects will be deleted after 7 days } -type StorageBackupURLs []struct { - Primary string `json:"primary"` - Backup string `json:"backup"` -} - -func Upload(input io.Reader, outputURI *url.URL, waitBetweenWrites, writeTimeout time.Duration, storageBackupURLs StorageBackupURLs) (*drivers.SaveDataOutput, error) { +func Upload(input io.Reader, outputURI *url.URL, waitBetweenWrites, writeTimeout time.Duration, storageBackupURLs map[string]string) (*drivers.SaveDataOutput, error) { if strings.HasSuffix(outputURI.Path, ".ts") || strings.HasSuffix(outputURI.Path, ".mp4") { // For segments we just write them in one go here and return early. // (Otherwise the incremental write logic below caused issues with clipping since it results in partial segments being written.) @@ -118,7 +113,7 @@ func Upload(input io.Reader, outputURI *url.URL, waitBetweenWrites, writeTimeout return nil, nil } -func uploadFileWithBackup(outputURI *url.URL, fileContents []byte, fields *drivers.FileProperties, writeTimeout time.Duration, withRetries bool, storageBackupURLs StorageBackupURLs) (out *drivers.SaveDataOutput, bytesWritten int64, err error) { +func uploadFileWithBackup(outputURI *url.URL, fileContents []byte, fields *drivers.FileProperties, writeTimeout time.Duration, withRetries bool, storageBackupURLs map[string]string) (out *drivers.SaveDataOutput, bytesWritten int64, err error) { out, bytesWritten, primaryErr := uploadFile(outputURI, fileContents, fields, writeTimeout, withRetries) if primaryErr == nil { return out, bytesWritten, nil @@ -134,11 +129,11 @@ func uploadFileWithBackup(outputURI *url.URL, fileContents []byte, fields *drive return uploadFile(backupURI, fileContents, fields, writeTimeout, withRetries) } -func buildBackupURI(outputURI *url.URL, storageBackupURLs StorageBackupURLs) (*url.URL, error) { +func buildBackupURI(outputURI *url.URL, storageBackupURLs map[string]string) (*url.URL, error) { outputURIStr := outputURI.String() - for _, strj := range storageBackupURLs { - if strings.HasPrefix(outputURIStr, strj.Primary) { - backupStr := strings.Replace(outputURIStr, strj.Primary, strj.Backup, 1) + for primary, backup := range storageBackupURLs { + if strings.HasPrefix(outputURIStr, primary) { + backupStr := strings.Replace(outputURIStr, primary, backup, 1) return url.Parse(backupStr) } } @@ -185,7 +180,7 @@ func uploadFile(outputURI *url.URL, fileContents []byte, fields *drivers.FilePro return out, bytesWritten, err } -func extractThumb(outputURI *url.URL, segment []byte, storageBackupURLs StorageBackupURLs) error { +func extractThumb(outputURI *url.URL, segment []byte, storageBackupURLs map[string]string) error { tmpDir, err := os.MkdirTemp(os.TempDir(), "thumb-*") if err != nil { return fmt.Errorf("temp file creation failed: %w", err)