From 313913737f13f97d9816dbc3d729e7bcd454a531 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Mon, 3 Jun 2024 16:25:49 +0800 Subject: [PATCH] RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057) try to fix #4052. --------- Co-authored-by: winlin --- trunk/conf/full.conf | 7 ++ trunk/doc/CHANGELOG.md | 1 + trunk/src/app/srs_app_config.cpp | 22 +++++- trunk/src/app/srs_app_config.hpp | 1 + trunk/src/app/srs_app_rtc_source.cpp | 37 +++++----- trunk/src/app/srs_app_rtc_source.hpp | 1 + trunk/src/core/srs_core_version5.hpp | 2 +- trunk/src/kernel/srs_kernel_codec.cpp | 99 ++++++++++++++++----------- trunk/src/kernel/srs_kernel_codec.hpp | 9 ++- trunk/src/kernel/srs_kernel_error.hpp | 3 +- trunk/src/utest/srs_utest_config.cpp | 23 +++++++ 11 files changed, 139 insertions(+), 66 deletions(-) diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index 3cdb3788f4..6c3f671e25 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -556,6 +556,13 @@ vhost rtc.vhost.srs.com { # Overwrite by env SRS_VHOST_RTC_KEEP_BFRAME for all vhosts. # default: off keep_bframe off; + # Whether to keep the h.264 SEI type NALU packet. + # DJI drone M30T will send many SEI type NALU packet, while iOS hardware decoder (Video Toolbox) + # dislike to feed it so many SEI NALU between NonIDR and IDR NALU packets. + # @see https://github.com/ossrs/srs/issues/4052 + # Overwrite by env SRS_VHOST_RTC_KEEP_AVC_NALU_SEI for all vhosts. + # Default: on + keep_avc_nalu_sei on; # The transcode audio bitrate, for RTMP to RTC. # Overwrite by env SRS_VHOST_RTC_OPUS_BITRATE for all vhosts. # [8000, 320000] diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index c211bb378f..9acfb5ce9d 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -7,6 +7,7 @@ The changelog for SRS. ## SRS 5.0 Changelog +* v5.0, 2024-06-03, Merge [#4057](https://github.com/ossrs/srs/pull/4057): RTC: Support dropping h.264 SEI from NALUs. v5.0.213 (#4057) * v5.0, 2024-04-23, Merge [#4038](https://github.com/ossrs/srs/pull/4038): RTMP: Do not response publish start message if hooks fail. v5.0.212 (#4038) * v5.0, 2024-04-22, Merge [#4033](https://github.com/ossrs/srs/pull/4033): issue #3967: support x509 certification chiain in single pem file. v5.0.211 (#4033) * v5.0, 2024-03-26, Filter JSONP callback function name. v5.0.210 diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index f6bf733b5c..c062429572 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2726,7 +2726,7 @@ srs_error_t SrsConfig::check_normal_config() && m != "bframe" && m != "aac" && m != "stun_timeout" && m != "stun_strict_check" && m != "dtls_role" && m != "dtls_version" && m != "drop_for_pt" && m != "rtc_to_rtmp" && m != "pli_for_rtmp" && m != "rtmp_to_rtc" && m != "keep_bframe" && m != "opus_bitrate" - && m != "aac_bitrate") { + && m != "aac_bitrate" && m != "keep_avc_nalu_sei") { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.rtc.%s of %s", m.c_str(), vhost->arg0().c_str()); } } @@ -4474,6 +4474,26 @@ bool SrsConfig::get_rtc_keep_bframe(string vhost) return SRS_CONF_PERFER_FALSE(conf->arg0()); } +bool SrsConfig::get_rtc_keep_avc_nalu_sei(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_BOOL2("srs.vhost.rtc.keep_avc_nalu_sei"); // SRS_VHOST_RTC_KEEP_AVC_NALU_SEI + + static bool DEFAULT = true; + + SrsConfDirective* conf = get_rtc(vhost); + + if (!conf) { + return DEFAULT; + } + + conf = conf->get("keep_avc_nalu_sei"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return SRS_CONF_PERFER_TRUE(conf->arg0()); +} + bool SrsConfig::get_rtc_from_rtmp(string vhost) { SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.rtc.rtmp_to_rtc"); // SRS_VHOST_RTC_RTMP_TO_RTC diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index 99d0d5e52c..0552c16dee 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -533,6 +533,7 @@ class SrsConfig SrsConfDirective* get_rtc(std::string vhost); bool get_rtc_enabled(std::string vhost); bool get_rtc_keep_bframe(std::string vhost); + bool get_rtc_keep_avc_nalu_sei(std::string vhost); bool get_rtc_from_rtmp(std::string vhost); srs_utime_t get_rtc_stun_timeout(std::string vhost); bool get_rtc_stun_strict_check(std::string vhost); diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 792f15dde0..8b1102d6a6 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -724,6 +724,7 @@ SrsRtcFromRtmpBridge::SrsRtcFromRtmpBridge(SrsRtcSource* source) latest_codec_ = SrsAudioCodecIdForbidden; rtmp_to_rtc = false; keep_bframe = false; + keep_avc_nalu_sei = true; merge_nalus = false; meta = new SrsMetaCache(); audio_sequence = 0; @@ -774,9 +775,10 @@ srs_error_t SrsRtcFromRtmpBridge::initialize(SrsRequest* r) } keep_bframe = _srs_config->get_rtc_keep_bframe(req->vhost); + keep_avc_nalu_sei = _srs_config->get_rtc_keep_avc_nalu_sei(req->vhost); merge_nalus = _srs_config->get_rtc_server_merge_nalus(); - srs_trace("RTC bridge from RTMP, rtmp2rtc=%d, keep_bframe=%d, merge_nalus=%d", - rtmp_to_rtc, keep_bframe, merge_nalus); + srs_trace("RTC bridge from RTMP, rtmp2rtc=%d, keep_bframe=%d, keep_avc_nalu_sei=%d, merge_nalus=%d", + rtmp_to_rtc, keep_bframe, keep_avc_nalu_sei, merge_nalus); return err; } @@ -1025,12 +1027,6 @@ srs_error_t SrsRtcFromRtmpBridge::on_video(SrsSharedPtrMessage* msg) for (int i = 0; i < nn_samples; i++) { SrsSample* sample = samples[i]; - // We always ignore bframe here, if config to discard bframe, - // the bframe flag will not be set. - if (sample->bframe) { - continue; - } - if (sample->size <= kRtpMaxPayloadSize) { if ((err = package_single_nalu(msg, sample, pkts)) != srs_success) { return srs_error_wrap(err, "package single nalu"); @@ -1062,14 +1058,27 @@ srs_error_t SrsRtcFromRtmpBridge::filter(SrsSharedPtrMessage* msg, SrsFormat* fo // Update samples to shared frame. for (int i = 0; i < format->video->nb_samples; ++i) { SrsSample* sample = &format->video->samples[i]; + + if (!keep_avc_nalu_sei && format->vcodec->id == SrsVideoCodecIdAVC) { + SrsAvcNaluType avc_nalu_type; + // TODO: FIXME use static method to parse avc nalu type. + if ((err = SrsVideoFrame::parse_avc_nalu_type(sample, avc_nalu_type)) != srs_success) { + return srs_error_wrap(err, "parse avc nalu_type"); + } + if (avc_nalu_type == SrsAvcNaluTypeSEI) { + // srs_warn("skip avc nalu type SEI, size=%d", sample->size); + continue; + } + } // Because RTC does not support B-frame, so we will drop them. // TODO: Drop B-frame in better way, which not cause picture corruption. - if (!keep_bframe) { - if ((err = sample->parse_bframe()) != srs_success) { + if (!keep_bframe && format->vcodec->id == SrsVideoCodecIdAVC) { + bool is_b_frame; + if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) { return srs_error_wrap(err, "parse bframe"); } - if (sample->bframe) { + if (is_b_frame) { continue; } } @@ -1149,12 +1158,6 @@ srs_error_t SrsRtcFromRtmpBridge::package_nalus(SrsSharedPtrMessage* msg, const for (int i = 0; i < (int)samples.size(); i++) { SrsSample* sample = samples[i]; - // We always ignore bframe here, if config to discard bframe, - // the bframe flag will not be set. - if (sample->bframe) { - continue; - } - if (!sample->size) { continue; } diff --git a/trunk/src/app/srs_app_rtc_source.hpp b/trunk/src/app/srs_app_rtc_source.hpp index e4fa643efb..6285d85d01 100644 --- a/trunk/src/app/srs_app_rtc_source.hpp +++ b/trunk/src/app/srs_app_rtc_source.hpp @@ -259,6 +259,7 @@ class SrsRtcFromRtmpBridge : public ISrsLiveSourceBridge SrsAudioCodecId latest_codec_; SrsAudioTranscoder* codec_; bool keep_bframe; + bool keep_avc_nalu_sei; bool merge_nalus; uint16_t audio_sequence; uint16_t video_sequence; diff --git a/trunk/src/core/srs_core_version5.hpp b/trunk/src/core/srs_core_version5.hpp index c8909af8de..8fe8d836d1 100644 --- a/trunk/src/core/srs_core_version5.hpp +++ b/trunk/src/core/srs_core_version5.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 5 #define VERSION_MINOR 0 -#define VERSION_REVISION 212 +#define VERSION_REVISION 213 #endif diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index c201a2bd24..5ef6eb80a5 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -403,63 +403,23 @@ SrsSample::SrsSample() { size = 0; bytes = NULL; - bframe = false; } SrsSample::SrsSample(char* b, int s) { size = s; bytes = b; - bframe = false; } SrsSample::~SrsSample() { } -srs_error_t SrsSample::parse_bframe() -{ - srs_error_t err = srs_success; - - uint8_t header = bytes[0]; - SrsAvcNaluType nal_type = (SrsAvcNaluType)(header & kNalTypeMask); - - if (nal_type != SrsAvcNaluTypeNonIDR && nal_type != SrsAvcNaluTypeDataPartitionA && nal_type != SrsAvcNaluTypeIDR) { - return err; - } - - SrsBuffer* stream = new SrsBuffer(bytes, size); - SrsAutoFree(SrsBuffer, stream); - - // Skip nalu header. - stream->skip(1); - - SrsBitBuffer bitstream(stream); - int32_t first_mb_in_slice = 0; - if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) { - return srs_error_wrap(err, "nalu read uev"); - } - - int32_t slice_type_v = 0; - if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) { - return srs_error_wrap(err, "nalu read uev"); - } - SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v; - - if (slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1) { - bframe = true; - srs_verbose("nal_type=%d, slice type=%d", nal_type, slice_type); - } - - return err; -} - SrsSample* SrsSample::copy() { SrsSample* p = new SrsSample(); p->bytes = bytes; p->size = size; - p->bframe = bframe; return p; } @@ -554,7 +514,6 @@ srs_error_t SrsFrame::add_sample(char* bytes, int size) SrsSample* sample = &samples[nb_samples++]; sample->bytes = bytes; sample->size = size; - sample->bframe = false; return err; } @@ -625,6 +584,64 @@ SrsVideoCodecConfig* SrsVideoFrame::vcodec() return (SrsVideoCodecConfig*)codec; } +srs_error_t SrsVideoFrame::parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type) +{ + srs_error_t err = srs_success; + + if (sample == NULL || sample->size < 1) { + return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu"); + } + + uint8_t header = sample->bytes[0]; + avc_nalu_type = (SrsAvcNaluType)(header & kNalTypeMask); + + return err; +} + +srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame) +{ + srs_error_t err = srs_success; + + if (sample == NULL || sample->size < 1) { + return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu"); + } + + SrsAvcNaluType nalu_type; + if ((err = parse_avc_nalu_type(sample, nalu_type)) != srs_success) { + return srs_error_wrap(err, "parse avc nalu type error"); + } + + if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA && nalu_type != SrsAvcNaluTypeIDR) { + is_b_frame = false; + return err; + } + + SrsBuffer* stream = new SrsBuffer(sample->bytes, sample->size); + SrsAutoFree(SrsBuffer, stream); + + // Skip nalu header. + stream->skip(1); + + SrsBitBuffer bitstream(stream); + int32_t first_mb_in_slice = 0; + if ((err = srs_avc_nalu_read_uev(&bitstream, first_mb_in_slice)) != srs_success) { + return srs_error_wrap(err, "nalu read uev"); + } + + int32_t slice_type_v = 0; + if ((err = srs_avc_nalu_read_uev(&bitstream, slice_type_v)) != srs_success) { + return srs_error_wrap(err, "nalu read uev"); + } + SrsAvcSliceType slice_type = (SrsAvcSliceType)slice_type_v; + + is_b_frame = slice_type == SrsAvcSliceTypeB || slice_type == SrsAvcSliceTypeB1; + if (is_b_frame) { + srs_verbose("nalu_type=%d, slice type=%d", nalu_type, slice_type); + } + + return err; +} + SrsFormat::SrsFormat() { acodec = NULL; diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 2b21f81e43..a652414bde 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -522,7 +522,7 @@ std::string srs_avc_level2str(SrsAvcLevel level); /** * A sample is the unit of frame. - * It's a NALU for H.264. + * It's a NALU for H.264, H.265. * It's the whole AAC raw data for AAC. * @remark Neither SPS/PPS or ASC is sample unit, it's codec sequence header. */ @@ -533,15 +533,11 @@ class SrsSample int size; // The ptr of unit, user must free it. char* bytes; - // Whether is B frame. - bool bframe; public: SrsSample(); SrsSample(char* b, int s); ~SrsSample(); public: - // If we need to know whether sample is bframe, we have to parse the NALU payload. - srs_error_t parse_bframe(); // Copy sample, share the bytes pointer. SrsSample* copy(); }; @@ -714,6 +710,9 @@ class SrsVideoFrame : public SrsFrame virtual srs_error_t add_sample(char* bytes, int size); public: virtual SrsVideoCodecConfig* vcodec(); +public: + static srs_error_t parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type); + static srs_error_t parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame); }; /** diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 26caa7b843..d52e7a9b67 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -272,7 +272,8 @@ XX(ERROR_INOTIFY_OPENFD , 3094, "InotifyOpenFd", "Failed to open inotify fd for config listener") \ XX(ERROR_INOTIFY_WATCH , 3095, "InotfyWatch", "Failed to watch inotify for config listener") \ XX(ERROR_HTTP_URL_UNESCAPE , 3096, "HttpUrlUnescape", "Failed to unescape URL for HTTP") \ - XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") + XX(ERROR_HTTP_WITH_BODY , 3097, "HttpWithBody", "Failed for HTTP body") \ + XX(ERROR_AVC_NALU_EMPTY , 3102, "AvcNaluEmpty", "AVC NALU is empty") /**************************************************/ /* HTTP/StreamConverter protocol error. */ diff --git a/trunk/src/utest/srs_utest_config.cpp b/trunk/src/utest/srs_utest_config.cpp index 0d2bdfdfe9..0860fab159 100644 --- a/trunk/src/utest/srs_utest_config.cpp +++ b/trunk/src/utest/srs_utest_config.cpp @@ -4402,6 +4402,29 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesVhostRtc) SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "on"); EXPECT_TRUE(conf.get_rtc_keep_bframe("__defaultVhost__")); + + { + // make sure the default value is false, if defined incorrect env value. + SrsSetEnvConfig(rtc_keep_bframe, "SRS_VHOST_RTC_KEEP_BFRAME", "onn"); + EXPECT_FALSE(conf.get_rtc_keep_bframe("__defaultVhost__")); + + } + + { + SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "off"); + EXPECT_FALSE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__")); + } + + { + SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "on"); + EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__")); + } + + { + // make sure the default value is true, if defined incorrect env value. + SrsSetEnvConfig(rtc_keep_avc_nalu_sei, "SRS_VHOST_RTC_KEEP_AVC_NALU_SEI", "xx"); + EXPECT_TRUE(conf.get_rtc_keep_avc_nalu_sei("__defaultVhost__")); + } } if (true) {