From 7306780f60c52e64abaf6486f144e494a90d4623 Mon Sep 17 00:00:00 2001 From: Max Holland Date: Thu, 12 Oct 2023 15:35:08 +0100 Subject: [PATCH] For media files just do a simple file write rather than incremental (#35) * For media files just do a simple file write rather than incremental The incremental writes caused issues with clipping since it results in partial segments being written. With this change the whole segment is written in one go * comment --- core/uploader.go | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/core/uploader.go b/core/uploader.go index 2d7b8b6..85d2d6a 100644 --- a/core/uploader.go +++ b/core/uploader.go @@ -23,6 +23,24 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t return err } + // While we wait for storj to implement an easier method for global object deletion we are hacking something + // here to allow us to have recording objects deleted after 7 days. + var fields *drivers.FileProperties + if strings.Contains(outputURI, "gateway.storjshare.io/catalyst-recordings-com") { + fields = &drivers.FileProperties{ + Metadata: map[string]string{ + "Object-Expires": "+168h", // Objects will be deleted after 7 days + }, + } + } + + if strings.HasSuffix(outputURI, ".ts") || strings.HasSuffix(outputURI, ".mp4") { + // For segments we just write them in one go here and return early. + // (Otherwise the incremental write logic below caused issues with clipping since it results in partial segments being written.) + _, err := session.SaveData(context.Background(), "", input, fields, writeTimeout) + return err + } + var fileContents = []byte{} var lastWrite = time.Now() @@ -49,7 +67,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t // Only write the latest version of the data that's been piped in if enough time has elapsed since the last write if lastWrite.Add(waitBetweenWrites).Before(time.Now()) { - if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), nil, writeTimeout); err != nil { + if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout); err != nil { // Just log this error, since it'll effectively be retried after the next interval log.Printf("Failed to write: %s", err) } else { @@ -62,17 +80,6 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t return err } - // While we wait for storj to implement an easier method for global object deletion we are hacking something - // here to allow us to have recording objects deleted after 7 days. - var fields *drivers.FileProperties - if strings.Contains(outputURI, "gateway.storjshare.io/catalyst-recordings-com") { - fields = &drivers.FileProperties{ - Metadata: map[string]string{ - "Object-Expires": "+168h", // Objects will be deleted after 7 days - }, - } - } - // We have to do this final write, otherwise there might be final data that's arrived since the last periodic write if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout); err != nil { // Don't ignore this error, since there won't be any further attempts to write