From c5b6b728763713d764ddda28ad52c50a99ba3638 Mon Sep 17 00:00:00 2001 From: Winlin Date: Fri, 4 Jul 2025 14:23:13 -0400 Subject: [PATCH] RTMP2RTC: Support dual video track for bridge. v7.0.44 (#4413) This PR refactors the RTMP to RTC bridge to support multiple video tracks by implementing lazy initialization of audio and video tracks. Instead of pre-determining track parameters during bridge construction, tracks are now initialized dynamically when the first packet of each type is received, allowing proper codec detection and track configuration for dual video track scenarios. Failed to view WHEP with HEVC before publishing RTMP, because the default codec is AVC and will not be updated until the stream is published. This enables better handling of streams with multiple video tracks in RTMP to WebRTC bridging scenarios. Now, you are able to: 1. View WHEP with HEVC: Play with WebRTC: http://localhost:8080/players/whep.html?schema=http&&codec=hevc 2. Then publish by RTMP: `ffmpeg -stream_loop -1 -re -i doc/source.flv -c:v libx265 -profile:v main -preset fast -b:v 2000k -maxrate 2000k -bufsize 2000k -bf 0 -c:a aac -b:a 48k -ar 44100 -ac 2 -f flv rtmp://localhost/live/livestream` Or publish RTMP with HEVC, then view by WHEP. Note that if the codecs do not match, the error log will display RTC: `Drop for ssrc xxxxxx not found`. For example, this can occur if you publish RTMP with HEVC while viewing the stream with AVC. --- trunk/doc/CHANGELOG.md | 1 + trunk/src/app/srs_app_rtc_source.cpp | 130 ++++++++++++++++++++---- trunk/src/app/srs_app_rtc_source.hpp | 13 ++- trunk/src/app/srs_app_stream_bridge.cpp | 59 +---------- trunk/src/app/srs_app_stream_bridge.hpp | 3 - trunk/src/core/srs_core_version7.hpp | 2 +- 6 files changed, 123 insertions(+), 85 deletions(-) diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index 1b046a4a7..7f8364966 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -7,6 +7,7 @@ The changelog for SRS. ## SRS 7.0 Changelog +* v7.0, 2025-07-04, Merge [#4413](https://github.com/ossrs/srs/pull/4413): RTMP2RTC: Support dual video track for bridge. v7.0.44 (#4413) * v7.0, 2025-07-03, Merge [#4349](https://github.com/ossrs/srs/pull/4349): rtc2rtmp: Support WebRTC-to-RTMP conversion with HEVC. v7.0.43 (#4349) * v7.0, 2025-06-04, Merge [#4310](https://github.com/ossrs/srs/pull/4310): Player: Get codec by webrtc api: pc.getStats. v7.0.42 (#4310) * v7.0, 2025-06-04, Merge [#4325](https://github.com/ossrs/srs/pull/4325): fix bug: loop transcoding #3516. v7.0.41 (#4325) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 7325760e7..0d5a8d944 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -467,22 +467,42 @@ void SrsRtcSource::init_for_play_before_publishing() audio_track_desc->media_ = new SrsAudioPayload(kAudioPayloadType, "opus", kAudioSamplerate, kAudioChannel); } - // video track description + // video track descriptions - support both H.264 and H.265 for play before publishing + // This allows clients to choose their preferred codec during SDP negotiation if (true) { - SrsRtcTrackDescription* video_track_desc = new SrsRtcTrackDescription(); - stream_desc->video_track_descs_.push_back(video_track_desc); + // H.264 track description + SrsRtcTrackDescription* h264_track_desc = new SrsRtcTrackDescription(); + stream_desc->video_track_descs_.push_back(h264_track_desc); - video_track_desc->type_ = "video"; - video_track_desc->id_ = "video-" + srs_random_str(8); + h264_track_desc->type_ = "video"; + h264_track_desc->id_ = "video-h264-" + srs_random_str(8); - uint32_t video_ssrc = SrsRtcSSRCGenerator::instance()->generate_ssrc(); - video_track_desc->ssrc_ = video_ssrc; - video_track_desc->direction_ = "recvonly"; + uint32_t h264_ssrc = SrsRtcSSRCGenerator::instance()->generate_ssrc(); + h264_track_desc->ssrc_ = h264_ssrc; + h264_track_desc->direction_ = "recvonly"; - SrsVideoPayload* video_payload = new SrsVideoPayload(kVideoPayloadType, "H264", kVideoSamplerate); - video_track_desc->media_ = video_payload; + SrsVideoPayload* h264_payload = new SrsVideoPayload(kVideoPayloadType, "H264", kVideoSamplerate); + h264_track_desc->media_ = h264_payload; - video_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"); + h264_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"); + } + + if (true) { + // H.265 track description + SrsRtcTrackDescription* h265_track_desc = new SrsRtcTrackDescription(); + stream_desc->video_track_descs_.push_back(h265_track_desc); + + h265_track_desc->type_ = "video"; + h265_track_desc->id_ = "video-h265-" + srs_random_str(8); + + uint32_t h265_ssrc = SrsRtcSSRCGenerator::instance()->generate_ssrc(); + h265_track_desc->ssrc_ = h265_ssrc; + h265_track_desc->direction_ = "recvonly"; + + SrsVideoPayload* h265_payload = new SrsVideoPayload(KVideoPayloadTypeHevc, "H265", kVideoSamplerate); + h265_track_desc->media_ = h265_payload; + + h265_payload->set_h265_param_desc("level-id=156;profile-id=1;tier-flag=0;tx-mode=SRST"); } set_stream_desc(stream_desc.get()); @@ -838,10 +858,12 @@ srs_error_t SrsRtcSource::on_timer(srs_utime_t interval) #ifdef SRS_FFMPEG_FIT -SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc, uint8_t apt, uint32_t vssrc, uint8_t vpt) +SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, SrsSharedPtr source) { - req = NULL; bridge_ = bridge; + source_ = source; + + req = NULL; format = new SrsRtmpFormat(); codec_ = new SrsAudioTranscoder(); latest_codec_ = SrsAudioCodecIdForbidden; @@ -852,10 +874,15 @@ SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc, audio_sequence = 0; video_sequence = 0; - audio_ssrc_ = assrc; - audio_payload_type_ = apt; - video_ssrc_ = vssrc; - video_payload_type_ = vpt; + // Initialize with default values - will be set during lazy initialization + audio_ssrc_ = 0; + audio_payload_type_ = 0; + video_ssrc_ = 0; + video_payload_type_ = 0; + + // Lazy initialization flags + audio_initialized_ = false; + video_initialized_ = false; } SrsRtcRtpBuilder::~SrsRtcRtpBuilder() @@ -865,6 +892,53 @@ SrsRtcRtpBuilder::~SrsRtcRtpBuilder() srs_freep(meta); } +srs_error_t SrsRtcRtpBuilder::initialize_audio_track(SrsAudioCodecId codec) +{ + srs_error_t err = srs_success; + + // Get the audio track description for the specified codec, as we will always + // transcode to opus for WebRTC. + std::string codec_name = "opus"; + std::vector descs = source_->get_track_desc("audio", "opus"); + + if (!descs.empty()) { + // Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079 + SrsRtcTrackDescription* track = descs.at(0); + audio_ssrc_ = track->ssrc_; + audio_payload_type_ = track->media_->pt_; + } else { + audio_payload_type_ = kAudioPayloadType; + } + + srs_trace("RTMP2RTC: Initialize audio track for %s with codec=%s, ssrc=%u, pt=%d", + srs_audio_codec_id2str(codec).c_str(), codec_name.c_str(), audio_ssrc_, audio_payload_type_); + + return err; +} + +srs_error_t SrsRtcRtpBuilder::initialize_video_track(SrsVideoCodecId codec) +{ + srs_error_t err = srs_success; + + // Get the video track description for the detected codec + std::string codec_name = srs_video_codec_id2str(codec); + std::vector descs = source_->get_track_desc("video", codec_name); + + if (!descs.empty()) { + // Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079 + SrsRtcTrackDescription* track = descs.at(0); + video_ssrc_ = track->ssrc_; + video_payload_type_ = track->media_->pt_; + } else { + video_payload_type_ = kVideoPayloadType; + } + + srs_trace("RTMP2RTC: Initialize video track with codec=%s, ssrc=%u, pt=%d", + codec_name.c_str(), video_ssrc_, video_payload_type_); + + return err; +} + srs_error_t SrsRtcRtpBuilder::initialize(SrsRequest* r) { srs_error_t err = srs_success; @@ -942,6 +1016,14 @@ srs_error_t SrsRtcRtpBuilder::on_audio(SrsSharedPtrMessage* msg) return err; } + // Initialize audio track on first packet with actual codec + if (!audio_initialized_) { + if ((err = initialize_audio_track(acodec)) != srs_success) { + return srs_error_wrap(err, "init audio track"); + } + audio_initialized_ = true; + } + // ignore sequence header srs_assert(format->audio); @@ -997,10 +1079,10 @@ srs_error_t SrsRtcRtpBuilder::init_codec(SrsAudioCodecId codec) // Update the latest codec in stream. if (latest_codec_ == SrsAudioCodecIdForbidden) { - srs_trace("RTMP2RTC: Init audio codec to %d(%s)", codec, srs_audio_codec_id2str(codec).c_str()); + srs_trace("RTMP2RTC: Init audio transcoder codec to %d(%s)", codec, srs_audio_codec_id2str(codec).c_str()); } else { - srs_trace("RTMP2RTC: Switch audio codec %d(%s) to %d(%s)", latest_codec_, srs_audio_codec_id2str(latest_codec_).c_str(), - codec, srs_audio_codec_id2str(codec).c_str()); + srs_trace("RTMP2RTC: Switch audio transcoder codec %d(%s) to %d(%s)", latest_codec_, srs_audio_codec_id2str(latest_codec_).c_str(), + codec, srs_audio_codec_id2str(codec).c_str()); } latest_codec_ = codec; @@ -1097,8 +1179,12 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) return err; } - if ((err = bridge_->update_codec(vcodec)) != srs_success) { - return srs_error_wrap(err, "update codec"); + // Initialize video track on first packet with actual codec + if (!video_initialized_) { + if ((err = initialize_video_track(vcodec)) != srs_success) { + return srs_error_wrap(err, "init video track"); + } + video_initialized_ = true; } bool has_idr = false; diff --git a/trunk/src/app/srs_app_rtc_source.hpp b/trunk/src/app/srs_app_rtc_source.hpp index 4822fbe62..535d59ef9 100644 --- a/trunk/src/app/srs_app_rtc_source.hpp +++ b/trunk/src/app/srs_app_rtc_source.hpp @@ -48,6 +48,8 @@ class SrsLiveSource; const int kAudioPayloadType = 111; // Firefox defaults as 126, Chrome is 102. const int kVideoPayloadType = 102; +// Chrome HEVC defaults as 49. +const int KVideoPayloadTypeHevc = 49; class SrsNtp { @@ -287,9 +289,18 @@ private: uint32_t video_ssrc_; uint8_t audio_payload_type_; uint8_t video_payload_type_; +private: + SrsSharedPtr source_; + // Lazy initialization flags + bool audio_initialized_; + bool video_initialized_; public: - SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc, uint8_t apt, uint32_t vssrc, uint8_t vpt); + SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, SrsSharedPtr source); virtual ~SrsRtcRtpBuilder(); +private: + // Lazy initialization methods + srs_error_t initialize_audio_track(SrsAudioCodecId codec); + srs_error_t initialize_video_track(SrsVideoCodecId codec); public: virtual srs_error_t initialize(SrsRequest* r); virtual srs_error_t on_publish(); diff --git a/trunk/src/app/srs_app_stream_bridge.cpp b/trunk/src/app/srs_app_stream_bridge.cpp index 80e0b999b..1a06ce365 100644 --- a/trunk/src/app/srs_app_stream_bridge.cpp +++ b/trunk/src/app/srs_app_stream_bridge.cpp @@ -68,35 +68,8 @@ SrsFrameToRtcBridge::SrsFrameToRtcBridge(SrsSharedPtr source) source_ = source; #if defined(SRS_FFMPEG_FIT) - uint32_t audio_ssrc = 0; - uint8_t audio_payload_type = 0; - uint32_t video_ssrc = 0; - uint8_t video_payload_type = 0; - - // audio track ssrc - if (true) { - std::vector descs = source->get_track_desc("audio", "opus"); - if (!descs.empty()) { - audio_ssrc = descs.at(0)->ssrc_; - } - // Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079 - audio_payload_type = descs.empty() ? kAudioPayloadType : descs.front()->media_->pt_; - } - - // video track ssrc - if (true) { - std::vector descs = source->get_track_desc("video", ""); - if (!descs.empty()) { - video_ssrc = descs.at(0)->ssrc_; - } - // Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079 - video_payload_type = descs.empty() ? kVideoPayloadType : descs.front()->media_->pt_; - } - - rtp_builder_ = new SrsRtcRtpBuilder(this, audio_ssrc, audio_payload_type, video_ssrc, video_payload_type); + rtp_builder_ = new SrsRtcRtpBuilder(this, source); #endif - - video_codec_id_ = SrsVideoCodecIdReserved; } SrsFrameToRtcBridge::~SrsFrameToRtcBridge() @@ -158,37 +131,7 @@ srs_error_t SrsFrameToRtcBridge::on_rtp(SrsRtpPacket* pkt) return source_->on_rtp(pkt); } -srs_error_t SrsFrameToRtcBridge::update_codec(SrsVideoCodecId id) -{ - srs_error_t err = srs_success; - if (video_codec_id_ == id) { - return err; - } - - std::vector video_track_descs = source_->get_track_desc("video", ""); - if (video_track_descs.empty()) { - return srs_error_new(ERROR_RTC_NO_TRACK, "no track found for conversion"); - } - - SrsRtcTrackDescription* video_track_desc = video_track_descs.at(0); - SrsVideoPayload* video_payload = (SrsVideoPayload*)video_track_desc->media_; - - if (id == SrsVideoCodecIdHEVC) { - video_payload->name_ = "H265"; - video_payload->set_h265_param_desc("level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST"); - } else { - video_payload->name_ = "H264"; - video_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"); - } - - srs_trace("RTC: Switch video codec %d(%s) to %d(%s)", video_codec_id_, srs_video_codec_id2str(video_codec_id_).c_str(), - id, srs_video_codec_id2str(id).c_str()); - - video_codec_id_ = id; - - return err; -} #endif diff --git a/trunk/src/app/srs_app_stream_bridge.hpp b/trunk/src/app/srs_app_stream_bridge.hpp index d5391c97c..dfa4857c8 100644 --- a/trunk/src/app/srs_app_stream_bridge.hpp +++ b/trunk/src/app/srs_app_stream_bridge.hpp @@ -65,8 +65,6 @@ private: #if defined(SRS_FFMPEG_FIT) SrsRtcRtpBuilder* rtp_builder_; #endif -private: - SrsVideoCodecId video_codec_id_; public: SrsFrameToRtcBridge(SrsSharedPtr source); virtual ~SrsFrameToRtcBridge(); @@ -76,7 +74,6 @@ public: virtual void on_unpublish(); virtual srs_error_t on_frame(SrsSharedPtrMessage* frame); srs_error_t on_rtp(SrsRtpPacket* pkt); - srs_error_t update_codec(SrsVideoCodecId id); }; #endif diff --git a/trunk/src/core/srs_core_version7.hpp b/trunk/src/core/srs_core_version7.hpp index 3824a2a3a..a836beea1 100644 --- a/trunk/src/core/srs_core_version7.hpp +++ b/trunk/src/core/srs_core_version7.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 7 #define VERSION_MINOR 0 -#define VERSION_REVISION 43 +#define VERSION_REVISION 44 #endif \ No newline at end of file