Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

transmux: use smaller chunks when concatenating long list of files #1212

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 39 additions & 1 deletion video/transmux.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (

const (
Mp4DurationLimit = 21600 //MP4s will be generated only for first 6 hours
MaxArgLimit = 250
)

func MuxTStoMP4(tsInputFile, mp4OutputFile string) ([]string, error) {
Expand Down Expand Up @@ -141,6 +142,29 @@ func ConcatTS(tsFileName string, segmentsList *TSegmentList, sourceMediaPlaylist
break
}
}
// If the argument list of files gets too long, linux might complain about exceeding
// MAX_ARG limit and ffmpeg (or any other command) using the long list will fail to run.
// So we split into chunked files then concat it one final time to get the final file.
if len(segmentFilenames) > MaxArgLimit {
chunks := ConcatChunkedFiles(segmentFilenames, MaxArgLimit)

var chunkFiles []string
for idx, chunk := range chunks {
concatArg := "concat:" + strings.Join(chunk, "|")
chunkFilename := fileBaseWithoutExt + "_" + "chunk" + strconv.Itoa(idx) + ".ts"
chunkFiles = append(chunkFiles, chunkFilename)
err := concatFiles(concatArg, chunkFilename)
if err != nil {
return totalBytes, fmt.Errorf("failed to file-concat a chunk (#%d)into a ts file: %w", idx, err)
}
}
if len(chunkFiles) == 0 {
return totalBytes, fmt.Errorf("failed to generate chunks to concat")
}
// override with the chunkFilenames instead
segmentFilenames = chunkFiles

}
concatArg := "concat:" + strings.Join(segmentFilenames, "|")

// Use file-based concatenation by reading segment files in text file
Expand All @@ -150,7 +174,6 @@ func ConcatTS(tsFileName string, segmentsList *TSegmentList, sourceMediaPlaylist
}

return totalBytes, nil

} else {
// Create a text file containing filenames of the segments
segmentListTxtFileName := fileBaseWithoutExt + ".txt"
Expand Down Expand Up @@ -255,3 +278,18 @@ func concatFiles(segmentList, outputTsFileName string) error {
}
return nil
}

// ConcatChunkedFiles splits the segmentFilenames into smaller chunks based on the maxLength value,
// where maxLength is the maximum number of filenames per chunk.
func ConcatChunkedFiles(filenames []string, maxLength int) [][]string {
var chunks [][]string
for maxLength > 0 && len(filenames) > 0 {
if len(filenames) <= maxLength {
chunks = append(chunks, filenames)
break
}
chunks = append(chunks, filenames[:maxLength])
filenames = filenames[maxLength:]
}
return chunks
}
49 changes: 49 additions & 0 deletions video/transmux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"github.com/stretchr/testify/require"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
)
Expand Down Expand Up @@ -213,6 +215,53 @@ func TestItConcatsStreamsOnlyUptoMP4DurationLimit(t *testing.T) {
require.Equal(t, int64(406268), totalBytesW)
}

func TestConcatChunkedFiles(t *testing.T) {
filenames := make([]string, 10)
for i := range filenames {
filenames[i] = "file" + strconv.Itoa(i+1)
}

testCases := []struct {
name string
maxLength int
wantChunks [][]string
}{
{
name: "MaxLengthLessThanLength",
maxLength: 3,
wantChunks: [][]string{
{"file1", "file2", "file3"},
{"file4", "file5", "file6"},
{"file7", "file8", "file9"},
{"file10"},
},
},
{
name: "MaxLengthEqualToLength",
maxLength: 10,
wantChunks: [][]string{
{"file1", "file2", "file3", "file4", "file5", "file6", "file7", "file8", "file9", "file10"},
},
},
{
name: "MaxLengthGreaterThanLength",
maxLength: 15,
wantChunks: [][]string{
{"file1", "file2", "file3", "file4", "file5", "file6", "file7", "file8", "file9", "file10"},
},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
gotChunks := ConcatChunkedFiles(filenames, tc.maxLength)
if !reflect.DeepEqual(gotChunks, tc.wantChunks) {
t.Errorf("ConcatChunkedFiles(%v, %d) = %v, want %v", filenames, tc.maxLength, gotChunks, tc.wantChunks)
}
})
}
}

func populateRenditionSegmentList() *TRenditionList {
segmentFiles := []string{"../test/fixtures/seg-0.ts", "../test/fixtures/seg-1.ts", "../test/fixtures/seg-2.ts"}

Expand Down
Loading