Revert "RTC: Support dropping h.264 SEI from NALUs. v5.0.213 v6.0.125 (…

…ossrs#4057)" This reverts commit 1656391.
winlinvip · Jun 13, 2024 · 88a9cc8 · 88a9cc8
1 parent c8fa269
commit 88a9cc8
Show file tree

Hide file tree

Showing 12 changed files with 66 additions and 141 deletions.
diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf
@@ -562,13 +562,6 @@ vhost rtc.vhost.srs.com {
         # Overwrite by env SRS_VHOST_RTC_KEEP_BFRAME for all vhosts.
         # default: off
         keep_bframe off;
-        # Whether to keep the h.264 SEI type NALU packet.
-        # DJI drone M30T will send many SEI type NALU packet, while iOS hardware decoder (Video Toolbox)
-        # dislike to feed it so many SEI NALU between NonIDR and IDR NALU packets.
-        # @see https://github.com/ossrs/srs/issues/4052
-        # Overwrite by env SRS_VHOST_RTC_KEEP_AVC_NALU_SEI for all vhosts.
-        # Default: on
-        keep_avc_nalu_sei on;
         # The transcode audio bitrate, for RTMP to RTC.
         # Overwrite by env SRS_VHOST_RTC_OPUS_BITRATE for all vhosts.
         # [8000, 320000]

diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md
@@ -7,7 +7,6 @@ The changelog for SRS.
 <a name="v6-changes"></a>
 
 ## SRS 6.0 Changelog
-* v6.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v6.0.125 (#4057)
 * v6.0, 2024-04-26, Merge [#4044](https://github.com/ossrs/srs/pull/4044): fix: correct SRS_ERRNO_MAP_HTTP duplicate error code. v6.0.124 (#4044)
 * v6.0, 2024-04-23, Merge [#4038](https://github.com/ossrs/srs/pull/4038): RTMP: Do not response publish start message if hooks fail. v6.0.123 (#4038)
 * v6.0, 2024-04-22, Merge [#4033](https://github.com/ossrs/srs/pull/4033): issue #3967: support x509 certification chiain in single pem file. v6.0.122 (#4033)
@@ -136,7 +135,6 @@ The changelog for SRS.
 <a name="v5-changes"></a>
 
 ## SRS 5.0 Changelog
-* v5.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057)
 * v5.0, 2024-04-23, Merge [#4038](https://github.com/ossrs/srs/pull/4038): RTMP: Do not response publish start message if hooks fail. v5.0.212 (#4038)
 * v5.0, 2024-04-22, Merge [#4033](https://github.com/ossrs/srs/pull/4033): issue #3967: support x509 certification chiain in single pem file. v5.0.211 (#4033)
 * v5.0, 2024-03-26, Filter JSONP callback function name. v5.0.210

diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp
@@ -2726,7 +2726,7 @@ srs_error_t SrsConfig::check_normal_config()
                         && m != "bframe" && m != "aac" && m != "stun_timeout" && m != "stun_strict_check"
                         && m != "dtls_role" && m != "dtls_version" && m != "drop_for_pt" && m != "rtc_to_rtmp"
                         && m != "pli_for_rtmp" && m != "rtmp_to_rtc" && m != "keep_bframe" && m != "opus_bitrate"
-                        && m != "aac_bitrate" && m != "keep_avc_nalu_sei") {
+                        && m != "aac_bitrate") {
                         return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.rtc.%s of %s", m.c_str(), vhost->arg0().c_str());
                     }
                 }
@@ -4474,26 +4474,6 @@ bool SrsConfig::get_rtc_keep_bframe(string vhost)
     return SRS_CONF_PERFER_FALSE(conf->arg0());
 }
 
-bool SrsConfig::get_rtc_keep_avc_nalu_sei(std::string vhost)
-{
-    SRS_OVERWRITE_BY_ENV_BOOL2("srs.vhost.rtc.keep_avc_nalu_sei"); // SRS_VHOST_RTC_KEEP_AVC_NALU_SEI
-
-    static bool DEFAULT = true;
-
-    SrsConfDirective* conf = get_rtc(vhost);
-
-    if (!conf) {
-        return DEFAULT;
-    }
-
-    conf = conf->get("keep_avc_nalu_sei");
-    if (!conf || conf->arg0().empty()) {
-        return DEFAULT;
-    }
-
-    return SRS_CONF_PERFER_TRUE(conf->arg0());
-}
-
 bool SrsConfig::get_rtc_from_rtmp(string vhost)
 {
     SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.rtmp_to_rtc"); // SRS_VHOST_RTC_RTMP_TO_RTC

diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp
@@ -533,7 +533,6 @@ class SrsConfig
     SrsConfDirective* get_rtc(std::string vhost);
     bool get_rtc_enabled(std::string vhost);
     bool get_rtc_keep_bframe(std::string vhost);
-    bool get_rtc_keep_avc_nalu_sei(std::string vhost);
     bool get_rtc_from_rtmp(std::string vhost);
     srs_utime_t get_rtc_stun_timeout(std::string vhost);
     bool get_rtc_stun_strict_check(std::string vhost);

diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp
@@ -739,7 +739,6 @@ SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc,
     codec_ = new SrsAudioTranscoder();
     latest_codec_ = SrsAudioCodecIdForbidden;
     keep_bframe = false;
-    keep_avc_nalu_sei = true;
     merge_nalus = false;
     meta = new SrsMetaCache();
     audio_sequence = 0;
@@ -772,10 +771,8 @@ srs_error_t SrsRtcRtpBuilder::initialize(SrsRequest* r)
     format->try_annexb_first = _srs_config->try_annexb_first(r->vhost);
 
     keep_bframe = _srs_config->get_rtc_keep_bframe(req->vhost);
-    keep_avc_nalu_sei = _srs_config->get_rtc_keep_avc_nalu_sei(req->vhost);
     merge_nalus = _srs_config->get_rtc_server_merge_nalus();
-    srs_trace("RTC bridge from RTMP, keep_bframe=%d, keep_avc_nalu_sei=%d, merge_nalus=%d",
-        keep_bframe, keep_avc_nalu_sei, merge_nalus);
+    srs_trace("RTC bridge from RTMP, keep_bframe=%d, merge_nalus=%d", keep_bframe, merge_nalus);
 
     return err;
 }
@@ -1016,6 +1013,12 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg)
         for (int i = 0; i < nn_samples; i++) {
             SrsSample* sample = samples[i];
 
+            // We always ignore bframe here, if config to discard bframe,
+            // the bframe flag will not be set.
+            if (sample->bframe) {
+                continue;
+            }
+
             if (sample->size <= kRtpMaxPayloadSize) {
                 if ((err = package_single_nalu(msg, sample, pkts)) != srs_success) {
                     return srs_error_wrap(err, "package single nalu");
@@ -1047,27 +1050,14 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format
     // Update samples to shared frame.
     for (int i = 0; i < format->video->nb_samples; ++i) {
         SrsSample* sample = &format->video->samples[i];
-
-        if (!keep_avc_nalu_sei && format->vcodec->id == SrsVideoCodecIdAVC) {
-            SrsAvcNaluType avc_nalu_type;
-            // TODO: FIXME use static method to parse avc nalu type.
-            if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) {
-                return srs_error_wrap(err, "parse avc nalu_type");
-            }
-            if (avc_nalu_type == SrsAvcNaluTypeSEI) {
-                // srs_warn("skip avc nalu type SEI, size=%d", sample->size);
-                continue;
-            }
-        }
 
         // Because RTC does not support B-frame, so we will drop them.
         // TODO: Drop B-frame in better way, which not cause picture corruption.
-        if (!keep_bframe && format->vcodec->id == SrsVideoCodecIdAVC) {
-            bool is_b_frame;
-            if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) {
+        if (!keep_bframe) {
+            if ((err = sample->parse_bframe()) != srs_success) {
                 return srs_error_wrap(err, "parse bframe");
             }
-            if (is_b_frame) {
+            if (sample->bframe) {
                 continue;
             }
         }
@@ -1147,6 +1137,12 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect
     for (int i = 0; i < (int)samples.size(); i++) {
         SrsSample* sample = samples[i];
 
+        // We always ignore bframe here, if config to discard bframe,
+        // the bframe flag will not be set.
+        if (sample->bframe) {
+            continue;
+        }
+
         if (!sample->size) {
             continue;
         }

diff --git a/trunk/src/app/srs_app_rtc_source.hpp b/trunk/src/app/srs_app_rtc_source.hpp
@@ -262,7 +262,6 @@ class SrsRtcRtpBuilder
     SrsAudioCodecId latest_codec_;
     SrsAudioTranscoder* codec_;
     bool keep_bframe;
-    bool keep_avc_nalu_sei;
     bool merge_nalus;
     uint16_t audio_sequence;
     uint16_t video_sequence;

diff --git a/trunk/src/core/srs_core_version5.hpp b/trunk/src/core/srs_core_version5.hpp
@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    213
+#define VERSION_REVISION    212
 
 #endif
diff --git a/trunk/src/core/srs_core_version6.hpp b/trunk/src/core/srs_core_version6.hpp
@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       6
 #define VERSION_MINOR       0
-#define VERSION_REVISION    125
+#define VERSION_REVISION    124
 
 #endif
diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp
@@ -504,23 +504,63 @@ SrsSample::SrsSample()
 {
     size = 0;
     bytes = NULL;
+    bframe = false;
 }
 
 SrsSample::SrsSample(char* b, int s)
 {
     size = s;
     bytes = b;
+    bframe = false;
 }
 
 SrsSample::~SrsSample()
 {
 }
 
+srs_error_t SrsSample::parse_bframe()
+{
+    srs_error_t err = srs_success;
+
+    uint8_t header = bytes[0];
+    SrsAvcNaluType nal_type = (SrsAvcNaluType)(header & kNalTypeMask);
+
+    if (nal_type != SrsAvcNaluTypeNonIDR && nal_type != SrsAvcNaluTypeDataPartitionA && nal_type != SrsAvcNaluTypeIDR) {
+        return err;
+    }
+
+    SrsBuffer* stream = new SrsBuffer(bytes, size);
+    SrsAutoFree(SrsBuffer, stream);
+
+    // Skip nalu header.
+    stream->skip(1);
+
+    SrsBitBuffer bitstream(stream);
+    int32_t first_mb_in_slice = 0;
+    if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) {
+        return srs_error_wrap(err, "nalu read uev");
+    }
+
+    int32_t slice_type_v = 0;
+    if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) {
+        return srs_error_wrap(err, "nalu read uev");
+    }
+    SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v;
+
+    if (slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1) {
+        bframe = true;
+        srs_verbose("nal_type=%d, slice type=%d", nal_type, slice_type);
+    }
+
+    return err;
+}
+
 SrsSample* SrsSample::copy()
 {
     SrsSample* p = new SrsSample();
     p->bytes = bytes;
     p->size = size;
+    p->bframe = bframe;
     return p;
 }
 
@@ -615,6 +655,7 @@ srs_error_t SrsFrame::add_sample(char* bytes, int size)
     SrsSample* sample = &samples[nb_samples++];
     sample->bytes = bytes;
     sample->size = size;
+    sample->bframe = false;
 
     return err;
 }
@@ -698,64 +739,6 @@ SrsVideoCodecConfig* SrsVideoFrame::vcodec()
     return (SrsVideoCodecConfig*)codec;
 }
 
-srs_error_t SrsVideoFrame::parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type)
-{
-    srs_error_t err = srs_success;
-
-    if (sample == NULL || sample->size < 1) {
-        return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu");
-    }
-
-    uint8_t header = sample->bytes[0];
-    avc_nalu_type = (SrsAvcNaluType)(header & kNalTypeMask);
-
-    return err;
-}
-
-srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame)
-{
-    srs_error_t err = srs_success;
-
-    if (sample == NULL || sample->size < 1) {
-        return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu");
-    }
-
-    SrsAvcNaluType nalu_type;
-    if ((err = parse_avc_nalu_type(sample, nalu_type)) != srs_success) {
-        return srs_error_wrap(err, "parse avc nalu type error");
-    }
-
-    if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA && nalu_type != SrsAvcNaluTypeIDR) {
-        is_b_frame = false;
-        return err;
-    }
-
-    SrsBuffer* stream = new SrsBuffer(sample->bytes, sample->size);
-    SrsAutoFree(SrsBuffer, stream);
-
-    // Skip nalu header.
-    stream->skip(1);
-
-    SrsBitBuffer bitstream(stream);
-    int32_t first_mb_in_slice = 0;
-    if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) {
-        return srs_error_wrap(err, "nalu read uev");
-    }
-
-    int32_t slice_type_v = 0;
-    if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) {
-        return srs_error_wrap(err, "nalu read uev");
-    }
-    SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v;
-
-    is_b_frame = slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1;
-    if (is_b_frame) {
-        srs_verbose("nalu_type=%d, slice type=%d", nalu_type, slice_type);
-    }
-
-    return err;
-}
-
 SrsFormat::SrsFormat()
 {
     acodec = NULL;

diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp
@@ -1120,7 +1120,7 @@ std::string srs_hevc_level2str(SrsHevcLevel level);
 
 /**
  * A sample is the unit of frame.
- * It's a NALU for H.264, H.265.
+ * It's a NALU for H.264.
  * It's the whole AAC raw data for AAC.
  * @remark Neither SPS/PPS or ASC is sample unit, it's codec sequence header.
  */
@@ -1131,11 +1131,15 @@ class SrsSample
     int size;
     // The ptr of unit, user must free it.
     char* bytes;
+    // Whether is B frame.
+    bool bframe;
 public:
     SrsSample();
     SrsSample(char* b, int s);
     ~SrsSample();
 public:
+    // If we need to know whether sample is bframe, we have to parse the NALU payload.
+    srs_error_t parse_bframe();
     // Copy sample, share the bytes pointer.
     SrsSample* copy();
 };
@@ -1318,9 +1322,6 @@ class SrsVideoFrame : public SrsFrame
     virtual srs_error_t add_sample(char* bytes, int size);
 public:
     virtual SrsVideoCodecConfig* vcodec();
-public:
-    static srs_error_t parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type);
-    static srs_error_t parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame);
 };
 
 /**

diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp
@@ -276,8 +276,7 @@
     XX(ERROR_HEVC_DISABLED                 , 3098, "HevcDisabled", "HEVC is disabled") \
     XX(ERROR_HEVC_DECODE_ERROR             , 3099, "HevcDecode", "HEVC decode av stream failed")  \
     XX(ERROR_MP4_HVCC_CHANGE               , 3100, "Mp4HvcCChange", "MP4 does not support video HvcC change") \
-    XX(ERROR_HEVC_API_NO_PREFIXED          , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder") \
-    XX(ERROR_AVC_NALU_EMPTY                , 3102, "AvcNaluEmpty", "AVC NALU is empty")
+    XX(ERROR_HEVC_API_NO_PREFIXED          , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder")
 
 /**************************************************/
 /* HTTP/StreamConverter protocol error. */

diff --git a/trunk/src/utest/srs_utest_config.cpp b/trunk/src/utest/srs_utest_config.cpp
@@ -4405,29 +4405,6 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesVhostRtc)
 
         SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "on");
         EXPECT_TRUE(conf.get_rtc_keep_bframe("__defaultVhost__"));
-
-        {
-            // make sure the default value is false, if defined incorrect env value.
-            SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "onn");
-            EXPECT_FALSE(conf.get_rtc_keep_bframe("__defaultVhost__"));
-
-        }
-
-        {
-            SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "off");
-            EXPECT_FALSE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
-        }
-
-        {
-            SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "on");
-            EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
-        }
-
-        {
-            // make sure the default value is true, if defined incorrect env value.
-            SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "xx");
-            EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__"));
-        }
     }
 
     if (true) {