Skip to content

Commit

Permalink
ffmpeg: Fix frame dropping in filtergraph at EOF.
Browse files Browse the repository at this point in the history 
This is done by setting the EOF timestamp of the filtergraph
such that the timestamp is *after* the last frame inserted.

Also add tests exhibiting various cases uncovered in the process
of fixing this.
  • Loading branch information
@j0sh
j0sh committed Sep 17, 2019
1 parent 0246c39 commit d722afc
Show file tree
Hide file tree
Showing 2 changed files with 326 additions and 13 deletions.
322 changes: 311 additions & 11 deletions ffmpeg/ffmpeg_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ func setupTest(t *testing.T) (func(cmd string), string) {
// The script is passed two arguments:
// a tempdir and the current working directory.
cmdFunc := func(cmd string) {
cmd = "cd $0 && set -eux;\n" + cmd
out, err := exec.Command("bash", "-c", cmd, dir, wd).CombinedOutput()
if err != nil {
t.Error(string(out[:]))
Expand Down Expand Up @@ -370,8 +371,8 @@ func TestTranscoder_Timestamp(t *testing.T) {
ffprobe -loglevel warning -select_streams v -show_streams -count_frames out0test.ts > test.out
grep avg_frame_rate=30 test.out
grep r_frame_rate=30 test.out
grep nb_read_frames=28 test.out
grep duration_ts=84000 test.out
grep nb_read_frames=29 test.out
grep duration_ts=87000 test.out
grep start_pts=138000 test.out
`
run(cmd)
Expand Down Expand Up @@ -490,7 +491,7 @@ func TestTranscoder_Statistics_Encoded(t *testing.T) {
profiles := []VideoProfile{P240p30fps16x9, P144p30fps16x9, p144p60fps, podd123fps}
out := make([]TranscodeOptions, len(profiles))
for i, p := range profiles {
out[i] = TranscodeOptions{Profile: p, Oname: fmt.Sprintf("%s/out%d.mp4", dir, i)}
out[i] = TranscodeOptions{Profile: p, Oname: fmt.Sprintf("%s/out%d.ts", dir, i)}
}

res, err := Transcode3(&TranscodeOptionsIn{Fname: dir + "/test.ts"}, out)
Expand All @@ -510,7 +511,19 @@ func TestTranscoder_Statistics_Encoded(t *testing.T) {
}
// Since this is a 1-second input we should ideally have count of frames
if r.Frames != int(out[i].Profile.Framerate) {
t.Error("Mismatched frame counts")

// Some "special" cases (already have test cases covering these)
if p144p60fps == out[i].Profile {
if r.Frames != int(out[i].Profile.Framerate)+1 {
t.Error("Mismatched frame counts for 60fps; expected 61 frames but got ", r.Frames)
}
} else if podd123fps == out[i].Profile {
if r.Frames != 125 {
t.Error("Mismatched frame counts for 123fps; expected 125 frames but got ", r.Frames)
}
} else {
t.Error("Mismatched frame counts ", r.Frames, out[i].Profile.Framerate)
}
}

// Check frame counts against ffprobe-reported output
Expand All @@ -521,11 +534,9 @@ func TestTranscoder_Statistics_Encoded(t *testing.T) {
t.Error(err)
}
b := bufio.NewWriter(f)
fmt.Fprintf(b, `[STREAM]
width=%d
fmt.Fprintf(b, `width=%d
height=%d
nb_read_frames=%d
[/STREAM]
`, w, h, r.Frames)
b.Flush()
f.Close()
Expand All @@ -536,8 +547,8 @@ nb_read_frames=%d
fname=out%d
ffprobe -loglevel warning -hide_banner -count_frames -select_streams v -show_entries stream=width,height,nb_read_frames $fname.mp4 > $fname.stats
ls -lha
ffprobe -loglevel warning -hide_banner -count_frames -count_packets -select_streams v -show_streams 2>&1 $fname.ts | grep '^width=\|^height=\|nb_read_frames=' > $fname.stats
diff -u $fname.stats $fname.res.stats
`, i)

Expand All @@ -564,7 +575,7 @@ func TestTranscoder_StatisticsAspectRatio(t *testing.T) {
run(cmd)

// This will be adjusted to 124x70 by the rescaler (since source is 16:9)
pAdj := VideoProfile{Resolution: "124x456", Framerate: 15, Bitrate: "100k"}
pAdj := VideoProfile{Resolution: "124x456", Framerate: 16, Bitrate: "100k"}
out := []TranscodeOptions{TranscodeOptions{Profile: pAdj, Oname: dir + "/adj.mp4"}}
res, err := Transcode3(&TranscodeOptionsIn{Fname: dir + "/test.ts"}, out)
if err != nil || len(res.Encoded) <= 0 {
Expand Down Expand Up @@ -868,7 +879,7 @@ func TestTranscoder_Drop(t *testing.T) {
if err != nil {
t.Error(err)
}
if res.Decoded.Frames != 30 || res.Encoded[0].Frames != 29 { // XXX 29 ?!?
if res.Decoded.Frames != 30 || res.Encoded[0].Frames != 30 {
t.Error("Unexpected encoded/decoded frame counts ", res.Decoded.Frames, res.Encoded[0].Frames)
}
in.Fname = dir + "/novideo.ts"
Expand Down Expand Up @@ -1037,3 +1048,292 @@ func TestTranscoder_StreamCopyAndDrop(t *testing.T) {

}
}

func TestTranscoder_RepeatedTranscodes(t *testing.T) {

// We have an issue where for certain inputs, we get a few more frames
// if trying to transcode to the same framerate.
// This test is to ensure that those errors don't compound.

run, dir := setupTest(t)
defer os.RemoveAll(dir)

cmd := `
ffmpeg -i "$1"/../transcoder/test.ts -an -c:v copy -t 1 test-short.ts
ffprobe -count_frames -show_streams test-short.ts > test.stats
grep nb_read_frames=62 test.stats
# this is informative: including audio makes things a bit more "expected"
ffmpeg -i "$1"/../transcoder/test.ts -c:a copy -c:v copy -t 1 test-short-with-audio.ts
ffprobe -count_frames -show_streams -select_streams v test-short-with-audio.ts > test-with-audio.stats
grep nb_read_frames=60 test-with-audio.stats
`
run(cmd)

// Initial set up; convert 60fps input to 30fps (1 second's worth)
in := &TranscodeOptionsIn{Fname: dir + "/test-short.ts"}
out := []TranscodeOptions{TranscodeOptions{Oname: dir + "/0.ts", Profile: P144p30fps16x9}}
res, err := Transcode3(in, out)
if err != nil || res.Decoded.Frames != 62 || res.Encoded[0].Frames != 31 {
t.Error("Unexpected preconditions ", err, res)
}
frames := res.Encoded[0].Frames

// Transcode results repeatedly, ensuring we have the same results each time
for i := 0; i < 5; i++ {
in.Fname = fmt.Sprintf("%s/%d.ts", dir, i)
out[0].Oname = fmt.Sprintf("%s/%d.ts", dir, i+1)
res, err = Transcode3(in, out)
if err != nil ||
res.Decoded.Frames != frames || res.Encoded[0].Frames != frames {
t.Error(fmt.Sprintf("Unexpected frame count for input %d : decoded %d encoded %d", i, res.Decoded.Frames, res.Encoded[0].Frames))
}
if res.Decoded.Frames != 31 && res.Encoded[0].Frames != 31 {
t.Error("Unexpected frame count! ", res)
}
}

// Do the same but with audio. This yields a 30fps file.
in = &TranscodeOptionsIn{Fname: dir + "/test-short-with-audio.ts"}
out = []TranscodeOptions{TranscodeOptions{Oname: dir + "/audio-0.ts", Profile: P144p30fps16x9}}
res, err = Transcode3(in, out)
if err != nil || res.Decoded.Frames != 60 || res.Encoded[0].Frames != 30 {
t.Error("Unexpected preconditions ", err, res)
}
frames = res.Encoded[0].Frames

// Transcode results repeatedly, ensuring we have the same results each time
for i := 0; i < 5; i++ {
in.Fname = fmt.Sprintf("%s/audio-%d.ts", dir, i)
out[0].Oname = fmt.Sprintf("%s/audio-%d.ts", dir, i+1)
res, err = Transcode3(in, out)
if err != nil ||
res.Decoded.Frames != frames || res.Encoded[0].Frames != frames {
t.Error(fmt.Sprintf("Unexpected frame count for input %d : decoded %d encoded %d", i, res.Decoded.Frames, res.Encoded[0].Frames))
}
}
}

func TestTranscoder_MismatchedEncodeDecode(t *testing.T) {
// Encoded frame count does not match decoded frame count for mp4
// Note this is not an issue for mpegts! (this is sanity checked)

run, dir := setupTest(t)
defer os.RemoveAll(dir)

p144p60fps := P144p30fps16x9
p144p60fps.Framerate = 60

cmd := `
set -eux
cd $0
# prepare 1-second input
cp "$1/../transcoder/test.ts" inp.ts
ffmpeg -loglevel warning -i inp.ts -c:a copy -c:v copy -t 1 test.ts
`
run(cmd)

in := &TranscodeOptionsIn{Fname: dir + "/test.ts"}
out := []TranscodeOptions{TranscodeOptions{Oname: dir + "/out.mp4", Profile: p144p60fps}}
res, err := Transcode3(in, out)
if err != nil {
t.Error(err)
}
in.Fname = dir + "/out.mp4"
res2, err := Transcode3(in, nil)
if err != nil {
t.Error(err)
}
// TODO Ideally these two should match. As far as I can tell it is due
// to timestamp rounding around EOF. Note this does not happen with
// mpegts formatted output!
if res2.Decoded.Frames != 60 || res.Encoded[0].Frames != 61 {
t.Error("Did not get expected frame counts: is this fixed?!? ",
res2.Decoded.Frames, res.Encoded[0].Frames)
}
// Interestingly enough, ffmpeg shows that we have 61 packets but 60 frames.
// That indicates we are receving, encoding and muxing 61 *things* but one
// of them isn't a complete frame.
cmd = `
ffprobe -count_frames -show_packets -show_streams -select_streams v out.mp4 2>&1 > mp4.out
grep nb_read_frames=60 mp4.out
grep nb_read_packets=61 mp4.out
`
run(cmd)

// Sanity check mpegts works as expected
in.Fname = dir + "/test.ts"
out[0].Oname = dir + "/out.ts"
res, err = Transcode3(in, out)
if err != nil {
t.Error(err)
}
in.Fname = dir + "/out.ts"
res2, err = Transcode3(in, nil)
if err != nil {
t.Error(err)
}
if res2.Decoded.Frames != 61 || res.Encoded[0].Frames != 61 {
t.Error("Did not get expected frame counts for mpegts ",
res2.Decoded.Frames, res.Encoded[0].Frames)
}

// Sanity check we still get the same results with multiple outputs?
in.Fname = dir + "/test.ts"
out = []TranscodeOptions{
TranscodeOptions{Oname: dir + "/out2.ts", Profile: p144p60fps},
TranscodeOptions{Oname: dir + "/out2.mp4", Profile: p144p60fps},
}
res, err = Transcode3(in, out)
if err != nil {
t.Error(err)
}
in.Fname = dir + "/out2.ts"
res2, err = Transcode3(in, nil)
if err != nil {
t.Error(err)
}
in.Fname = dir + "/out2.mp4"
res3, err := Transcode3(in, nil)
if err != nil {
t.Error(err)
}
// first output is mpegts
if res2.Decoded.Frames != 61 || res.Encoded[0].Frames != 61 {
t.Error("Sanity check of mpegts failed ", res2.Decoded.Frames, res.Encoded[0].Frames)
}
if res3.Decoded.Frames != 60 || res.Encoded[1].Frames != 61 {
t.Error("Sanity check of mp4 failed ", res2.Decoded.Frames, res.Encoded[1].Frames)
}
}

func TestTranscoder_FFmpegMatching(t *testing.T) {
// Sanity check that ffmpeg matches the following behavior:

// No audio case
// 1 second input, N fps ( M frames; N != M for $reasons )
// Output set to N/2 fps . Output contains ( M / 2 ) frames

// Other cases with audio result in frame count matching FPS
// 1 second input, N fps ( N frames )
// Output set to M fps. Output contains M frames

// Weird framerate case
// 1 second input, N fps ( N frames )
// Output set to 123 fps. Output contains 125 frames

run, dir := setupTest(t)
defer os.RemoveAll(dir)

cmd := `
cp "$1/../transcoder/test.ts" test.ts
# Test no audio case
ffmpeg -loglevel warning -i test.ts -an -c:v copy -t 1 test-noaudio.ts
# sanity check one stream, and that no audio still has 60fps
ffprobe -count_frames -show_streams -show_format test-noaudio.ts 2>&1 > noaudio.stats
grep avg_frame_rate=60 noaudio.stats
grep nb_read_frames=62 noaudio.stats
grep nb_streams=1 noaudio.stats
grep codec_name=h264 noaudio.stats
ffmpeg -loglevel warning -i test.ts -c:a copy -c:v copy -t 1 test-60fps.ts
ffprobe -show_streams test-60fps.ts | grep avg_frame_rate=60
ls -lha
`
run(cmd)

checkStatsFile := func(in *TranscodeOptionsIn, out *TranscodeOptions, res *TranscodeResults) {
// Generate stats file for given LPMS output
f, err := os.Create(dir + "/lpms.stats")
if err != nil {
t.Error(err)
}
defer f.Close()
w, h, err := VideoProfileResolution(out.Profile)
if err != nil {
t.Error("Invalid profile ", err)
}
b := bufio.NewWriter(f)
fmt.Fprintf(b, `width=%d
height=%d
nb_read_frames=%d
`,
w, h, res.Encoded[0].Frames)
b.Flush()

// Run a ffmpeg command that attempts to match the given encode settings
run(fmt.Sprintf(`ffmpeg -loglevel warning -hide_banner -i %s -vsync passthrough -c:a aac -ar 44100 -ac 2 -c:v libx264 -vf fps=%d/1,scale=%dx%d -y ffmpeg.ts`, in.Fname, out.Profile.Framerate, w, h))

// Gather some ffprobe stats on the output of the above ffmpeg command
run(`ffprobe -loglevel warning -hide_banner -count_frames -select_streams v -show_streams 2>&1 ffmpeg.ts | grep '^width=\|^height=\|nb_read_frames=' > ffmpeg.stats`)
// Gather ffprobe stats on the output of the LPMS transcode itself
run(fmt.Sprintf(`ffprobe -loglevel warning -hide_banner -count_frames -select_streams v -show_streams 2>&1 %s | grep '^width=\|^height=\|nb_read_frames=' > ffprobe-lpms.stats`, out.Oname))

// Now ensure stats match across 3 files:
// 1. ffmpeg encoded results, as checked by ffprobe (ffmpeg.stats)
// 2. ffprobe-checked LPMS output (ffprobe-lpms.stats)
// 3. Statistics received directly from LPMS after transcoding (lpms.stat)
cmd := `
diff -u ffmpeg.stats lpms.stats
diff -u ffprobe-lpms.stats lpms.stats
`
run(cmd)
}

// no audio + 60fps input
in := &TranscodeOptionsIn{Fname: dir + "/test-noaudio.ts"}
out := []TranscodeOptions{TranscodeOptions{
Oname: dir + "/out-noaudio.ts",
Profile: P144p30fps16x9,
}}
res, err := Transcode3(in, out)
if err != nil {
t.Error(err)
}
if res.Encoded[0].Frames != 31 {
t.Error("Did not get expected frame count ", res.Encoded[0].Frames)
}
checkStatsFile(in, &out[0], res)

// audio + 60fps input, 30fps output, 30 frames actual
in.Fname = dir + "/test-60fps.ts"
out[0].Oname = dir + "/out-60fps-to-30fps.ts"
res, err = Transcode3(in, out)
if err != nil {
t.Error(err)
}
if res.Encoded[0].Frames != 30 {
t.Error("Did not get expected frame count ", res.Encoded[0].Frames)
}
checkStatsFile(in, &out[0], res)

// audio + 60fps input, 60fps output. 61 frames actual
in.Fname = dir + "/test-60fps.ts"
out[0].Oname = dir + "/out-60-to-120fps.ts"
out[0].Profile.Framerate = 60
res, err = Transcode3(in, out)
if err != nil {
t.Error(err)
}
if res.Encoded[0].Frames != 61 { //
t.Error("Did not get expected frame count ", res.Encoded[0].Frames)
}
checkStatsFile(in, &out[0], res)

// audio + 60fps input, 123 fps output. 125 frames actual
in.Fname = dir + "/test-60fps.ts"
out[0].Oname = dir + "/out-123fps.ts"
out[0].Profile.Framerate = 123
res, err = Transcode3(in, out)
if err != nil {
t.Error(err)
}
if res.Encoded[0].Frames != 125 { // TODO Find out why this isn't 123
t.Error("Did not get expected frame count ", res.Encoded[0].Frames)
}
checkStatsFile(in, &out[0], res)
}
Loading

0 comments on commit d722afc

Please sign in to comment.