RTMP2RTC: Support dual video track for bridge. v7.0.44 (#4413)
This PR refactors the RTMP to RTC bridge to support multiple video tracks by implementing lazy initialization of audio and video tracks. Instead of pre-determining track parameters during bridge construction, tracks are now initialized dynamically when the first packet of each type is received, allowing proper codec detection and track configuration for dual video track scenarios. Failed to view WHEP with HEVC before publishing RTMP, because the default codec is AVC and will not be updated until the stream is published. This enables better handling of streams with multiple video tracks in RTMP to WebRTC bridging scenarios. Now, you are able to: 1. View WHEP with HEVC: Play with WebRTC: http://localhost:8080/players/whep.html?schema=http&&codec=hevc 2. Then publish by RTMP: `ffmpeg -stream_loop -1 -re -i doc/source.flv -c:v libx265 -profile:v main -preset fast -b:v 2000k -maxrate 2000k -bufsize 2000k -bf 0 -c:a aac -b:a 48k -ar 44100 -ac 2 -f flv rtmp://localhost/live/livestream` Or publish RTMP with HEVC, then view by WHEP. Note that if the codecs do not match, the error log will display RTC: `Drop for ssrc xxxxxx not found`. For example, this can occur if you publish RTMP with HEVC while viewing the stream with AVC.
This commit is contained in:
Winlin
GitHub
parent
a19551540f
commit
c5b6b72876
|
|
@ -7,6 +7,7 @@ The changelog for SRS.
|
|||
<a name="v7-changes"></a>
|
||||
|
||||
## SRS 7.0 Changelog
|
||||
* v7.0, 2025-07-04, Merge [#4413](https://github.com/ossrs/srs/pull/4413): RTMP2RTC: Support dual video track for bridge. v7.0.44 (#4413)
|
||||
* v7.0, 2025-07-03, Merge [#4349](https://github.com/ossrs/srs/pull/4349): rtc2rtmp: Support WebRTC-to-RTMP conversion with HEVC. v7.0.43 (#4349)
|
||||
* v7.0, 2025-06-04, Merge [#4310](https://github.com/ossrs/srs/pull/4310): Player: Get codec by webrtc api: pc.getStats. v7.0.42 (#4310)
|
||||
* v7.0, 2025-06-04, Merge [#4325](https://github.com/ossrs/srs/pull/4325): fix bug: loop transcoding #3516. v7.0.41 (#4325)
|
||||
|
|
|
|||
|
|
@ -467,22 +467,42 @@ void SrsRtcSource::init_for_play_before_publishing()
|
|||
audio_track_desc->media_ = new SrsAudioPayload(kAudioPayloadType, "opus", kAudioSamplerate, kAudioChannel);
|
||||
}
|
||||
|
||||
// video track description
|
||||
// video track descriptions - support both H.264 and H.265 for play before publishing
|
||||
// This allows clients to choose their preferred codec during SDP negotiation
|
||||
if (true) {
|
||||
SrsRtcTrackDescription* video_track_desc = new SrsRtcTrackDescription();
|
||||
stream_desc->video_track_descs_.push_back(video_track_desc);
|
||||
// H.264 track description
|
||||
SrsRtcTrackDescription* h264_track_desc = new SrsRtcTrackDescription();
|
||||
stream_desc->video_track_descs_.push_back(h264_track_desc);
|
||||
|
||||
video_track_desc->type_ = "video";
|
||||
video_track_desc->id_ = "video-" + srs_random_str(8);
|
||||
h264_track_desc->type_ = "video";
|
||||
h264_track_desc->id_ = "video-h264-" + srs_random_str(8);
|
||||
|
||||
uint32_t video_ssrc = SrsRtcSSRCGenerator::instance()->generate_ssrc();
|
||||
video_track_desc->ssrc_ = video_ssrc;
|
||||
video_track_desc->direction_ = "recvonly";
|
||||
uint32_t h264_ssrc = SrsRtcSSRCGenerator::instance()->generate_ssrc();
|
||||
h264_track_desc->ssrc_ = h264_ssrc;
|
||||
h264_track_desc->direction_ = "recvonly";
|
||||
|
||||
SrsVideoPayload* video_payload = new SrsVideoPayload(kVideoPayloadType, "H264", kVideoSamplerate);
|
||||
video_track_desc->media_ = video_payload;
|
||||
SrsVideoPayload* h264_payload = new SrsVideoPayload(kVideoPayloadType, "H264", kVideoSamplerate);
|
||||
h264_track_desc->media_ = h264_payload;
|
||||
|
||||
video_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f");
|
||||
h264_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f");
|
||||
}
|
||||
|
||||
if (true) {
|
||||
// H.265 track description
|
||||
SrsRtcTrackDescription* h265_track_desc = new SrsRtcTrackDescription();
|
||||
stream_desc->video_track_descs_.push_back(h265_track_desc);
|
||||
|
||||
h265_track_desc->type_ = "video";
|
||||
h265_track_desc->id_ = "video-h265-" + srs_random_str(8);
|
||||
|
||||
uint32_t h265_ssrc = SrsRtcSSRCGenerator::instance()->generate_ssrc();
|
||||
h265_track_desc->ssrc_ = h265_ssrc;
|
||||
h265_track_desc->direction_ = "recvonly";
|
||||
|
||||
SrsVideoPayload* h265_payload = new SrsVideoPayload(KVideoPayloadTypeHevc, "H265", kVideoSamplerate);
|
||||
h265_track_desc->media_ = h265_payload;
|
||||
|
||||
h265_payload->set_h265_param_desc("level-id=156;profile-id=1;tier-flag=0;tx-mode=SRST");
|
||||
}
|
||||
|
||||
set_stream_desc(stream_desc.get());
|
||||
|
|
@ -838,10 +858,12 @@ srs_error_t SrsRtcSource::on_timer(srs_utime_t interval)
|
|||
|
||||
#ifdef SRS_FFMPEG_FIT
|
||||
|
||||
SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc, uint8_t apt, uint32_t vssrc, uint8_t vpt)
|
||||
SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, SrsSharedPtr<SrsRtcSource> source)
|
||||
{
|
||||
req = NULL;
|
||||
bridge_ = bridge;
|
||||
source_ = source;
|
||||
|
||||
req = NULL;
|
||||
format = new SrsRtmpFormat();
|
||||
codec_ = new SrsAudioTranscoder();
|
||||
latest_codec_ = SrsAudioCodecIdForbidden;
|
||||
|
|
@ -852,10 +874,15 @@ SrsRtcRtpBuilder::SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc,
|
|||
audio_sequence = 0;
|
||||
video_sequence = 0;
|
||||
|
||||
audio_ssrc_ = assrc;
|
||||
audio_payload_type_ = apt;
|
||||
video_ssrc_ = vssrc;
|
||||
video_payload_type_ = vpt;
|
||||
// Initialize with default values - will be set during lazy initialization
|
||||
audio_ssrc_ = 0;
|
||||
audio_payload_type_ = 0;
|
||||
video_ssrc_ = 0;
|
||||
video_payload_type_ = 0;
|
||||
|
||||
// Lazy initialization flags
|
||||
audio_initialized_ = false;
|
||||
video_initialized_ = false;
|
||||
}
|
||||
|
||||
SrsRtcRtpBuilder::~SrsRtcRtpBuilder()
|
||||
|
|
@ -865,6 +892,53 @@ SrsRtcRtpBuilder::~SrsRtcRtpBuilder()
|
|||
srs_freep(meta);
|
||||
}
|
||||
|
||||
srs_error_t SrsRtcRtpBuilder::initialize_audio_track(SrsAudioCodecId codec)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
// Get the audio track description for the specified codec, as we will always
|
||||
// transcode to opus for WebRTC.
|
||||
std::string codec_name = "opus";
|
||||
std::vector<SrsRtcTrackDescription*> descs = source_->get_track_desc("audio", "opus");
|
||||
|
||||
if (!descs.empty()) {
|
||||
// Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079
|
||||
SrsRtcTrackDescription* track = descs.at(0);
|
||||
audio_ssrc_ = track->ssrc_;
|
||||
audio_payload_type_ = track->media_->pt_;
|
||||
} else {
|
||||
audio_payload_type_ = kAudioPayloadType;
|
||||
}
|
||||
|
||||
srs_trace("RTMP2RTC: Initialize audio track for %s with codec=%s, ssrc=%u, pt=%d",
|
||||
srs_audio_codec_id2str(codec).c_str(), codec_name.c_str(), audio_ssrc_, audio_payload_type_);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsRtcRtpBuilder::initialize_video_track(SrsVideoCodecId codec)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
// Get the video track description for the detected codec
|
||||
std::string codec_name = srs_video_codec_id2str(codec);
|
||||
std::vector<SrsRtcTrackDescription*> descs = source_->get_track_desc("video", codec_name);
|
||||
|
||||
if (!descs.empty()) {
|
||||
// Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079
|
||||
SrsRtcTrackDescription* track = descs.at(0);
|
||||
video_ssrc_ = track->ssrc_;
|
||||
video_payload_type_ = track->media_->pt_;
|
||||
} else {
|
||||
video_payload_type_ = kVideoPayloadType;
|
||||
}
|
||||
|
||||
srs_trace("RTMP2RTC: Initialize video track with codec=%s, ssrc=%u, pt=%d",
|
||||
codec_name.c_str(), video_ssrc_, video_payload_type_);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
srs_error_t SrsRtcRtpBuilder::initialize(SrsRequest* r)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
|
@ -942,6 +1016,14 @@ srs_error_t SrsRtcRtpBuilder::on_audio(SrsSharedPtrMessage* msg)
|
|||
return err;
|
||||
}
|
||||
|
||||
// Initialize audio track on first packet with actual codec
|
||||
if (!audio_initialized_) {
|
||||
if ((err = initialize_audio_track(acodec)) != srs_success) {
|
||||
return srs_error_wrap(err, "init audio track");
|
||||
}
|
||||
audio_initialized_ = true;
|
||||
}
|
||||
|
||||
// ignore sequence header
|
||||
srs_assert(format->audio);
|
||||
|
||||
|
|
@ -997,10 +1079,10 @@ srs_error_t SrsRtcRtpBuilder::init_codec(SrsAudioCodecId codec)
|
|||
|
||||
// Update the latest codec in stream.
|
||||
if (latest_codec_ == SrsAudioCodecIdForbidden) {
|
||||
srs_trace("RTMP2RTC: Init audio codec to %d(%s)", codec, srs_audio_codec_id2str(codec).c_str());
|
||||
srs_trace("RTMP2RTC: Init audio transcoder codec to %d(%s)", codec, srs_audio_codec_id2str(codec).c_str());
|
||||
} else {
|
||||
srs_trace("RTMP2RTC: Switch audio codec %d(%s) to %d(%s)", latest_codec_, srs_audio_codec_id2str(latest_codec_).c_str(),
|
||||
codec, srs_audio_codec_id2str(codec).c_str());
|
||||
srs_trace("RTMP2RTC: Switch audio transcoder codec %d(%s) to %d(%s)", latest_codec_, srs_audio_codec_id2str(latest_codec_).c_str(),
|
||||
codec, srs_audio_codec_id2str(codec).c_str());
|
||||
}
|
||||
latest_codec_ = codec;
|
||||
|
||||
|
|
@ -1097,8 +1179,12 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg)
|
|||
return err;
|
||||
}
|
||||
|
||||
if ((err = bridge_->update_codec(vcodec)) != srs_success) {
|
||||
return srs_error_wrap(err, "update codec");
|
||||
// Initialize video track on first packet with actual codec
|
||||
if (!video_initialized_) {
|
||||
if ((err = initialize_video_track(vcodec)) != srs_success) {
|
||||
return srs_error_wrap(err, "init video track");
|
||||
}
|
||||
video_initialized_ = true;
|
||||
}
|
||||
|
||||
bool has_idr = false;
|
||||
|
|
|
|||
|
|
@ -48,6 +48,8 @@ class SrsLiveSource;
|
|||
const int kAudioPayloadType = 111;
|
||||
// Firefox defaults as 126, Chrome is 102.
|
||||
const int kVideoPayloadType = 102;
|
||||
// Chrome HEVC defaults as 49.
|
||||
const int KVideoPayloadTypeHevc = 49;
|
||||
|
||||
class SrsNtp
|
||||
{
|
||||
|
|
@ -287,9 +289,18 @@ private:
|
|||
uint32_t video_ssrc_;
|
||||
uint8_t audio_payload_type_;
|
||||
uint8_t video_payload_type_;
|
||||
private:
|
||||
SrsSharedPtr<SrsRtcSource> source_;
|
||||
// Lazy initialization flags
|
||||
bool audio_initialized_;
|
||||
bool video_initialized_;
|
||||
public:
|
||||
SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, uint32_t assrc, uint8_t apt, uint32_t vssrc, uint8_t vpt);
|
||||
SrsRtcRtpBuilder(SrsFrameToRtcBridge* bridge, SrsSharedPtr<SrsRtcSource> source);
|
||||
virtual ~SrsRtcRtpBuilder();
|
||||
private:
|
||||
// Lazy initialization methods
|
||||
srs_error_t initialize_audio_track(SrsAudioCodecId codec);
|
||||
srs_error_t initialize_video_track(SrsVideoCodecId codec);
|
||||
public:
|
||||
virtual srs_error_t initialize(SrsRequest* r);
|
||||
virtual srs_error_t on_publish();
|
||||
|
|
|
|||
|
|
@ -68,35 +68,8 @@ SrsFrameToRtcBridge::SrsFrameToRtcBridge(SrsSharedPtr<SrsRtcSource> source)
|
|||
source_ = source;
|
||||
|
||||
#if defined(SRS_FFMPEG_FIT)
|
||||
uint32_t audio_ssrc = 0;
|
||||
uint8_t audio_payload_type = 0;
|
||||
uint32_t video_ssrc = 0;
|
||||
uint8_t video_payload_type = 0;
|
||||
|
||||
// audio track ssrc
|
||||
if (true) {
|
||||
std::vector<SrsRtcTrackDescription*> descs = source->get_track_desc("audio", "opus");
|
||||
if (!descs.empty()) {
|
||||
audio_ssrc = descs.at(0)->ssrc_;
|
||||
}
|
||||
// Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079
|
||||
audio_payload_type = descs.empty() ? kAudioPayloadType : descs.front()->media_->pt_;
|
||||
}
|
||||
|
||||
// video track ssrc
|
||||
if (true) {
|
||||
std::vector<SrsRtcTrackDescription*> descs = source->get_track_desc("video", "");
|
||||
if (!descs.empty()) {
|
||||
video_ssrc = descs.at(0)->ssrc_;
|
||||
}
|
||||
// Note we must use the PT of source, see https://github.com/ossrs/srs/pull/3079
|
||||
video_payload_type = descs.empty() ? kVideoPayloadType : descs.front()->media_->pt_;
|
||||
}
|
||||
|
||||
rtp_builder_ = new SrsRtcRtpBuilder(this, audio_ssrc, audio_payload_type, video_ssrc, video_payload_type);
|
||||
rtp_builder_ = new SrsRtcRtpBuilder(this, source);
|
||||
#endif
|
||||
|
||||
video_codec_id_ = SrsVideoCodecIdReserved;
|
||||
}
|
||||
|
||||
SrsFrameToRtcBridge::~SrsFrameToRtcBridge()
|
||||
|
|
@ -158,37 +131,7 @@ srs_error_t SrsFrameToRtcBridge::on_rtp(SrsRtpPacket* pkt)
|
|||
return source_->on_rtp(pkt);
|
||||
}
|
||||
|
||||
srs_error_t SrsFrameToRtcBridge::update_codec(SrsVideoCodecId id)
|
||||
{
|
||||
srs_error_t err = srs_success;
|
||||
|
||||
if (video_codec_id_ == id) {
|
||||
return err;
|
||||
}
|
||||
|
||||
std::vector<SrsRtcTrackDescription*> video_track_descs = source_->get_track_desc("video", "");
|
||||
if (video_track_descs.empty()) {
|
||||
return srs_error_new(ERROR_RTC_NO_TRACK, "no track found for conversion");
|
||||
}
|
||||
|
||||
SrsRtcTrackDescription* video_track_desc = video_track_descs.at(0);
|
||||
SrsVideoPayload* video_payload = (SrsVideoPayload*)video_track_desc->media_;
|
||||
|
||||
if (id == SrsVideoCodecIdHEVC) {
|
||||
video_payload->name_ = "H265";
|
||||
video_payload->set_h265_param_desc("level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST");
|
||||
} else {
|
||||
video_payload->name_ = "H264";
|
||||
video_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f");
|
||||
}
|
||||
|
||||
srs_trace("RTC: Switch video codec %d(%s) to %d(%s)", video_codec_id_, srs_video_codec_id2str(video_codec_id_).c_str(),
|
||||
id, srs_video_codec_id2str(id).c_str());
|
||||
|
||||
video_codec_id_ = id;
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -65,8 +65,6 @@ private:
|
|||
#if defined(SRS_FFMPEG_FIT)
|
||||
SrsRtcRtpBuilder* rtp_builder_;
|
||||
#endif
|
||||
private:
|
||||
SrsVideoCodecId video_codec_id_;
|
||||
public:
|
||||
SrsFrameToRtcBridge(SrsSharedPtr<SrsRtcSource> source);
|
||||
virtual ~SrsFrameToRtcBridge();
|
||||
|
|
@ -76,7 +74,6 @@ public:
|
|||
virtual void on_unpublish();
|
||||
virtual srs_error_t on_frame(SrsSharedPtrMessage* frame);
|
||||
srs_error_t on_rtp(SrsRtpPacket* pkt);
|
||||
srs_error_t update_codec(SrsVideoCodecId id);
|
||||
};
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,6 @@
|
|||
|
||||
#define VERSION_MAJOR 7
|
||||
#define VERSION_MINOR 0
|
||||
#define VERSION_REVISION 43
|
||||
#define VERSION_REVISION 44
|
||||
|
||||
#endif
|
||||
Loading…
Reference in New Issue
Block a user