Skip to content

Commit

Permalink
For media files just do a simple file write rather than incremental
Browse files Browse the repository at this point in the history
The incremental writes caused issues with clipping since it results in partial segments being written. With this change the whole segment is written in one go
  • Loading branch information
mjh1 committed Oct 12, 2023
1 parent b299a0e commit cb465d1
Showing 1 changed file with 17 additions and 12 deletions.
29 changes: 17 additions & 12 deletions core/uploader.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,22 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t
return err
}

// While we wait for storj to implement an easier method for global object deletion we are hacking something
// here to allow us to have recording objects deleted after 7 days.
var fields *drivers.FileProperties
if strings.Contains(outputURI, "gateway.storjshare.io/catalyst-recordings-com") {
fields = &drivers.FileProperties{
Metadata: map[string]string{
"Object-Expires": "+168h", // Objects will be deleted after 7 days
},
}
}

Check warning on line 35 in core/uploader.go

View check run for this annotation

Codecov / codecov/patch

core/uploader.go#L30-L35

Added lines #L30 - L35 were not covered by tests

if strings.HasSuffix(outputURI, ".ts") || strings.HasSuffix(outputURI, ".mp4") {
_, err := session.SaveData(context.Background(), "", input, fields, writeTimeout)
return err
}

Check warning on line 40 in core/uploader.go

View check run for this annotation

Codecov / codecov/patch

core/uploader.go#L38-L40

Added lines #L38 - L40 were not covered by tests

var fileContents = []byte{}
var lastWrite = time.Now()

Expand All @@ -49,7 +65,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t

// Only write the latest version of the data that's been piped in if enough time has elapsed since the last write
if lastWrite.Add(waitBetweenWrites).Before(time.Now()) {
if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), nil, writeTimeout); err != nil {
if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout); err != nil {
// Just log this error, since it'll effectively be retried after the next interval
log.Printf("Failed to write: %s", err)
} else {
Expand All @@ -62,17 +78,6 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t
return err
}

// While we wait for storj to implement an easier method for global object deletion we are hacking something
// here to allow us to have recording objects deleted after 7 days.
var fields *drivers.FileProperties
if strings.Contains(outputURI, "gateway.storjshare.io/catalyst-recordings-com") {
fields = &drivers.FileProperties{
Metadata: map[string]string{
"Object-Expires": "+168h", // Objects will be deleted after 7 days
},
}
}

// We have to do this final write, otherwise there might be final data that's arrived since the last periodic write
if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout); err != nil {
// Don't ignore this error, since there won't be any further attempts to write
Expand Down

0 comments on commit cb465d1

Please sign in to comment.