From 56440413aa32a13cbfbb41a6d5ee611bae02ab2e Mon Sep 17 00:00:00 2001 From: Anthony Lu <67125539+koln67@users.noreply.github.com> Date: Wed, 28 Feb 2024 15:53:06 -0800 Subject: [PATCH] fix: use a better estimate of frame rate for cases with very short first sample durations (#838) Use the second sample in mp4 and webm formats. #835 had issues with merging due to golden file conflicts. Because we cannot make dependent pull requests, this is a replica of #835. --------- Signed-off-by: Cosmin Stejerean Co-authored-by: Cosmin Stejerean --- .../audio-video-with-two-trick-play/output.mpd | 4 ++-- .../output.m3u8 | 2 +- .../encryption-and-two-trick-plays/output.mpd | 4 ++-- .../testdata/live-profile-with-webm/output.m3u8 | 2 +- .../testdata/live-profile-with-webm/output.mpd | 2 +- .../opus-vp9-mp4-with-encryption/output.mpd | 2 +- .../testdata/vp8-mp4-with-encryption/output.mpd | 2 +- packager/app/test/testdata/vp8-webm/output.mpd | 2 +- .../testdata/vp9-webm-with-blockgroup/output.mpd | 2 +- packager/app/test/testdata/vp9-webm/output.mpd | 2 +- .../testdata/webm-subsample-encryption/output.mpd | 2 +- .../webm-vp9-full-sample-encryption/output.mpd | 2 +- .../test/testdata/webm-with-encryption/output.mpd | 2 +- packager/media/formats/mp2t/ts_muxer.cc | 3 +++ packager/media/formats/mp2t/ts_muxer.h | 4 ++-- packager/media/formats/mp4/segmenter.cc | 8 ++++++-- packager/media/formats/mp4/segmenter.h | 7 +++++-- packager/media/formats/webm/segmenter.cc | 14 +++++++++----- packager/media/formats/webm/segmenter.h | 5 ++++- 19 files changed, 44 insertions(+), 27 deletions(-) diff --git a/packager/app/test/testdata/audio-video-with-two-trick-play/output.mpd b/packager/app/test/testdata/audio-video-with-two-trick-play/output.mpd index dd692cffddd..e9c0ec69eb1 100644 --- a/packager/app/test/testdata/audio-video-with-two-trick-play/output.mpd +++ b/packager/app/test/testdata/audio-video-with-two-trick-play/output.mpd @@ -19,9 +19,9 @@ - + - + bear-640x360-video-trick_play_factor_2.mp4 diff --git a/packager/app/test/testdata/ec3-and-hls-single-segment-mp4-encrypted/output.m3u8 b/packager/app/test/testdata/ec3-and-hls-single-segment-mp4-encrypted/output.m3u8 index 7ec849768fe..5db89137c52 100644 --- a/packager/app/test/testdata/ec3-and-hls-single-segment-mp4-encrypted/output.m3u8 +++ b/packager/app/test/testdata/ec3-and-hls-single-segment-mp4-encrypted/output.m3u8 @@ -5,7 +5,7 @@ #EXT-X-MEDIA:TYPE=AUDIO,URI="bear-640x360-ec3-audio.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="2" -#EXT-X-STREAM-INF:BANDWIDTH=1174214,AVERAGE-BANDWIDTH=1061802,CODECS="avc1.64001e,ec-3",RESOLUTION=640x360,FRAME-RATE=9.990,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE +#EXT-X-STREAM-INF:BANDWIDTH=1174214,AVERAGE-BANDWIDTH=1061802,CODECS="avc1.64001e,ec-3",RESOLUTION=640x360,FRAME-RATE=29.970,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE bear-640x360-ec3-video.m3u8 #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=218705,AVERAGE-BANDWIDTH=159315,CODECS="avc1.64001e",RESOLUTION=640x360,CLOSED-CAPTIONS=NONE,URI="bear-640x360-ec3-video-iframe.m3u8" diff --git a/packager/app/test/testdata/encryption-and-two-trick-plays/output.mpd b/packager/app/test/testdata/encryption-and-two-trick-plays/output.mpd index f1e0f059722..88e1a8ef0cf 100644 --- a/packager/app/test/testdata/encryption-and-two-trick-plays/output.mpd +++ b/packager/app/test/testdata/encryption-and-two-trick-plays/output.mpd @@ -27,7 +27,7 @@ - + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== @@ -39,7 +39,7 @@ - + bear-640x360-video-trick_play_factor_2.mp4 diff --git a/packager/app/test/testdata/live-profile-with-webm/output.m3u8 b/packager/app/test/testdata/live-profile-with-webm/output.m3u8 index 57ad801cfc0..cfcfc457ead 100644 --- a/packager/app/test/testdata/live-profile-with-webm/output.m3u8 +++ b/packager/app/test/testdata/live-profile-with-webm/output.m3u8 @@ -5,5 +5,5 @@ #EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="2" -#EXT-X-STREAM-INF:BANDWIDTH=556353,AVERAGE-BANDWIDTH=412719,CODECS="vp08.00.10.08.01.02.02.02.00,vorbis",RESOLUTION=640x360,FRAME-RATE=30.303,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE +#EXT-X-STREAM-INF:BANDWIDTH=556353,AVERAGE-BANDWIDTH=412719,CODECS="vp08.00.10.08.01.02.02.02.00,vorbis",RESOLUTION=640x360,FRAME-RATE=29.412,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE stream_1.m3u8 diff --git a/packager/app/test/testdata/live-profile-with-webm/output.mpd b/packager/app/test/testdata/live-profile-with-webm/output.mpd index c41814ca158..ea922c3284c 100644 --- a/packager/app/test/testdata/live-profile-with-webm/output.mpd +++ b/packager/app/test/testdata/live-profile-with-webm/output.mpd @@ -14,7 +14,7 @@ - + diff --git a/packager/app/test/testdata/opus-vp9-mp4-with-encryption/output.mpd b/packager/app/test/testdata/opus-vp9-mp4-with-encryption/output.mpd index a9e318ea996..42125e65fd2 100644 --- a/packager/app/test/testdata/opus-vp9-mp4-with-encryption/output.mpd +++ b/packager/app/test/testdata/opus-vp9-mp4-with-encryption/output.mpd @@ -15,7 +15,7 @@ - + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== diff --git a/packager/app/test/testdata/vp8-mp4-with-encryption/output.mpd b/packager/app/test/testdata/vp8-mp4-with-encryption/output.mpd index d5d1caa5ee8..80dbde48de6 100644 --- a/packager/app/test/testdata/vp8-mp4-with-encryption/output.mpd +++ b/packager/app/test/testdata/vp8-mp4-with-encryption/output.mpd @@ -2,7 +2,7 @@ - + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== diff --git a/packager/app/test/testdata/vp8-webm/output.mpd b/packager/app/test/testdata/vp8-webm/output.mpd index 08d7c36986c..5eecef53b49 100644 --- a/packager/app/test/testdata/vp8-webm/output.mpd +++ b/packager/app/test/testdata/vp8-webm/output.mpd @@ -2,7 +2,7 @@ - + bear-640x360-video.webm diff --git a/packager/app/test/testdata/vp9-webm-with-blockgroup/output.mpd b/packager/app/test/testdata/vp9-webm-with-blockgroup/output.mpd index 4878206a730..c78caf168da 100644 --- a/packager/app/test/testdata/vp9-webm-with-blockgroup/output.mpd +++ b/packager/app/test/testdata/vp9-webm-with-blockgroup/output.mpd @@ -2,7 +2,7 @@ - + bear-vp9-blockgroup-video.webm diff --git a/packager/app/test/testdata/vp9-webm/output.mpd b/packager/app/test/testdata/vp9-webm/output.mpd index e87196a7328..5e109a04efe 100644 --- a/packager/app/test/testdata/vp9-webm/output.mpd +++ b/packager/app/test/testdata/vp9-webm/output.mpd @@ -11,7 +11,7 @@ - + bear-320x240-vp9-opus-video.webm diff --git a/packager/app/test/testdata/webm-subsample-encryption/output.mpd b/packager/app/test/testdata/webm-subsample-encryption/output.mpd index e700f5835ab..a68d23d32cd 100644 --- a/packager/app/test/testdata/webm-subsample-encryption/output.mpd +++ b/packager/app/test/testdata/webm-subsample-encryption/output.mpd @@ -2,7 +2,7 @@ - + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== diff --git a/packager/app/test/testdata/webm-vp9-full-sample-encryption/output.mpd b/packager/app/test/testdata/webm-vp9-full-sample-encryption/output.mpd index e700f5835ab..a68d23d32cd 100644 --- a/packager/app/test/testdata/webm-vp9-full-sample-encryption/output.mpd +++ b/packager/app/test/testdata/webm-vp9-full-sample-encryption/output.mpd @@ -2,7 +2,7 @@ - + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== diff --git a/packager/app/test/testdata/webm-with-encryption/output.mpd b/packager/app/test/testdata/webm-with-encryption/output.mpd index eebc46aba21..ee3390f9480 100644 --- a/packager/app/test/testdata/webm-with-encryption/output.mpd +++ b/packager/app/test/testdata/webm-with-encryption/output.mpd @@ -2,7 +2,7 @@ - + AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA== diff --git a/packager/media/formats/mp2t/ts_muxer.cc b/packager/media/formats/mp2t/ts_muxer.cc index b00de7c1e1d..c2a65f26531 100644 --- a/packager/media/formats/mp2t/ts_muxer.cc +++ b/packager/media/formats/mp2t/ts_muxer.cc @@ -49,6 +49,9 @@ Status TsMuxer::Finalize() { Status TsMuxer::AddMediaSample(size_t stream_id, const MediaSample& sample) { DCHECK_EQ(stream_id, 0u); + + // The duration of the first sample may have been adjusted, so use + // the duration of the second sample instead. if (num_samples_ < 2) { sample_durations_[num_samples_] = sample.duration() * kTsTimescale / streams().front()->time_scale(); diff --git a/packager/media/formats/mp2t/ts_muxer.h b/packager/media/formats/mp2t/ts_muxer.h index 145ed98be45..d184c52587c 100644 --- a/packager/media/formats/mp2t/ts_muxer.h +++ b/packager/media/formats/mp2t/ts_muxer.h @@ -38,8 +38,8 @@ class TsMuxer : public Muxer { void FireOnMediaEndEvent(); std::unique_ptr segmenter_; - int64_t sample_durations_[2]; - int64_t num_samples_ = 0; + int64_t sample_durations_[2] = {0, 0}; + size_t num_samples_ = 0; // Used in multi-segment mode for segment template. uint64_t segment_number_ = 0; diff --git a/packager/media/formats/mp4/segmenter.cc b/packager/media/formats/mp4/segmenter.cc index 0df3a81db9b..9313a0eb5e4 100644 --- a/packager/media/formats/mp4/segmenter.cc +++ b/packager/media/formats/mp4/segmenter.cc @@ -141,8 +141,12 @@ Status Segmenter::AddSample(size_t stream_id, const MediaSample& sample) { if (!status.ok()) return status; - if (sample_duration_ == 0) - sample_duration_ = sample.duration(); + // The duration of the first sample may have been adjusted, so use + // the duration of the second sample instead. + if (num_samples_ < 2) { + sample_durations_[num_samples_] = sample.duration(); + num_samples_++; + } stream_durations_[stream_id] += sample.duration(); return Status::OK; } diff --git a/packager/media/formats/mp4/segmenter.h b/packager/media/formats/mp4/segmenter.h index 5333ffe35c3..df488121222 100644 --- a/packager/media/formats/mp4/segmenter.h +++ b/packager/media/formats/mp4/segmenter.h @@ -99,7 +99,9 @@ class Segmenter { /// @return The sample duration in the timescale of the media. /// Returns 0 if no samples are added yet. - int32_t sample_duration() const { return sample_duration_; } + int64_t sample_duration() const { + return sample_durations_[num_samples_ < 2 ? 0 : 1]; + } protected: /// Update segmentation progress using ProgressListener. @@ -147,7 +149,8 @@ class Segmenter { ProgressListener* progress_listener_ = nullptr; uint64_t progress_target_ = 0u; uint64_t accumulated_progress_ = 0u; - int32_t sample_duration_ = 0; + int64_t sample_durations_[2] = {0, 0}; + size_t num_samples_ = 0; std::vector stream_durations_; std::vector key_frame_infos_; diff --git a/packager/media/formats/webm/segmenter.cc b/packager/media/formats/webm/segmenter.cc index fca31287b30..f05834c87d9 100644 --- a/packager/media/formats/webm/segmenter.cc +++ b/packager/media/formats/webm/segmenter.cc @@ -159,11 +159,15 @@ Status Segmenter::Finalize() { Status Segmenter::AddSample(const MediaSample& source_sample) { std::shared_ptr sample(source_sample.Clone()); - if (sample_duration_ == 0) { - first_timestamp_ = sample->pts(); - sample_duration_ = sample->duration(); - if (muxer_listener_) - muxer_listener_->OnSampleDurationReady(sample_duration_); + // The duration of the first sample may have been adjusted, so use + // the duration of the second sample instead. + if (num_samples_ < 2) { + sample_durations_[num_samples_] = sample->duration(); + if (num_samples_ == 0) + first_timestamp_ = sample->pts(); + else if (muxer_listener_) + muxer_listener_->OnSampleDurationReady(sample_durations_[num_samples_]); + num_samples_++; } UpdateProgress(sample->duration()); diff --git a/packager/media/formats/webm/segmenter.h b/packager/media/formats/webm/segmenter.h index 42dacc628eb..7a40cf73f50 100644 --- a/packager/media/formats/webm/segmenter.h +++ b/packager/media/formats/webm/segmenter.h @@ -139,8 +139,11 @@ class Segmenter { ProgressListener* progress_listener_ = nullptr; uint64_t progress_target_ = 0; uint64_t accumulated_progress_ = 0; + int64_t first_timestamp_ = 0; - int64_t sample_duration_ = 0; + int64_t sample_durations_[2] = {0, 0}; + size_t num_samples_ = 0; + // The position (in bytes) of the start of the Segment payload in the init // file. This is also the size of the header before the SeekHead. uint64_t segment_payload_pos_ = 0;