From 3cccef327c38e6b40567b7040bd7ff8f93a36e50 Mon Sep 17 00:00:00 2001 From: winlin Date: Sat, 11 Apr 2020 17:52:14 +0800 Subject: [PATCH] For #307, package opus when send it. --- trunk/src/app/srs_app_rtc.cpp | 46 ++++++--------------- trunk/src/app/srs_app_rtc.hpp | 5 --- trunk/src/app/srs_app_rtc_conn.cpp | 58 +++++++++++++++++++++++---- trunk/src/app/srs_app_rtc_conn.hpp | 6 +++ trunk/src/kernel/srs_kernel_codec.hpp | 4 +- trunk/src/kernel/srs_kernel_flv.cpp | 14 ++++++- trunk/src/kernel/srs_kernel_flv.hpp | 6 +++ 7 files changed, 90 insertions(+), 49 deletions(-) diff --git a/trunk/src/app/srs_app_rtc.cpp b/trunk/src/app/srs_app_rtc.cpp index ca525637e..7b74ba3fe 100644 --- a/trunk/src/app/srs_app_rtc.cpp +++ b/trunk/src/app/srs_app_rtc.cpp @@ -260,17 +260,12 @@ srs_error_t SrsRtpH264Muxer::packet_stap_a(const string &sps, const string& pps, SrsRtpOpusMuxer::SrsRtpOpusMuxer() { - sequence = 0; - timestamp = 0; transcode = NULL; } SrsRtpOpusMuxer::~SrsRtpOpusMuxer() { - if (transcode) { - delete transcode; - transcode = NULL; - } + srs_freep(transcode); } srs_error_t SrsRtpOpusMuxer::initialize() @@ -309,7 +304,7 @@ srs_error_t SrsRtpOpusMuxer::frame_to_packet(SrsSharedPtrMessage* shared_audio, } } - // Transcode aac packet to opus packets. + // Transcode an aac packet to many opus packets. SrsSample aac; aac.bytes = adts_audio; aac.size = nn_adts_audio; @@ -320,37 +315,20 @@ srs_error_t SrsRtpOpusMuxer::frame_to_packet(SrsSharedPtrMessage* shared_audio, return srs_error_wrap(err, "recode error"); } - // Package opus packets to RTP packets. - vector rtp_packets; + // Save OPUS packets in shared message. + if (nn_opus_packets <= 0) { + return err; + } + SrsSample samples[nn_opus_packets]; for (int i = 0; i < nn_opus_packets; i++) { - SrsSample sample; - sample.size = opus_sizes[i]; - sample.bytes = opus_payloads[i]; - if ((err = packet_opus(shared_audio, &sample, rtp_packets)) != srs_success) { - return srs_error_wrap(err, "packet as opus"); - } + SrsSample* p = samples + i; + p->size = opus_sizes[i]; + p->bytes = new char[p->size]; + memcpy(p->bytes, opus_payloads[i], p->size); } - shared_audio->set_rtp_packets(rtp_packets); - - return err; -} - -srs_error_t SrsRtpOpusMuxer::packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector& rtp_packets) -{ - srs_error_t err = srs_success; - - SrsRtpSharedPacket* packet = new SrsRtpSharedPacket(); - packet->rtp_header.set_marker(true); - if ((err = packet->create(timestamp, sequence++, kAudioSSRC, kOpusPayloadType, sample->bytes, sample->size)) != srs_success) { - return srs_error_wrap(err, "rtp packet encode"); - } - - // TODO: FIXME: Why 960? Need Refactoring? - timestamp += 960; - - rtp_packets.push_back(packet); + shared_audio->set_extra_payloads(samples, nn_opus_packets); return err; } diff --git a/trunk/src/app/srs_app_rtc.hpp b/trunk/src/app/srs_app_rtc.hpp index f409e2624..855c2e62b 100644 --- a/trunk/src/app/srs_app_rtc.hpp +++ b/trunk/src/app/srs_app_rtc.hpp @@ -90,9 +90,6 @@ private: class SrsRtpOpusMuxer { private: - // TODO: FIXME: How to handle timestamp overflow? - uint32_t timestamp; - uint16_t sequence; SrsAudioRecode* transcode; public: SrsRtpOpusMuxer(); @@ -100,8 +97,6 @@ public: virtual srs_error_t initialize(); public: srs_error_t frame_to_packet(SrsSharedPtrMessage* shared_audio, SrsFormat* format, char* adts_audio, int nn_adts_audio); -private: - srs_error_t packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector& rtp_packets); }; class SrsRtc diff --git a/trunk/src/app/srs_app_rtc_conn.cpp b/trunk/src/app/srs_app_rtc_conn.cpp index da3a4fc2f..4492d9122 100644 --- a/trunk/src/app/srs_app_rtc_conn.cpp +++ b/trunk/src/app/srs_app_rtc_conn.cpp @@ -442,6 +442,9 @@ SrsRtcSenderThread::SrsRtcSenderThread(SrsRtcSession* s, SrsUdpMuxSocket* u, int rtc_session = s; sendonly_ukt = u->copy_sendonly(); + + timestamp = 0; + sequence = 0; } SrsRtcSenderThread::~SrsRtcSenderThread() @@ -575,6 +578,8 @@ void SrsRtcSenderThread::update_sendonly_socket(SrsUdpMuxSocket* skt) void SrsRtcSenderThread::send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs, SrsUdpMuxSocket* skt, int* pnn, int* pnn_rtp_pkts) { + srs_error_t err = srs_success; + if (!rtc_session->dtls_session) { return; } @@ -584,14 +589,35 @@ void SrsRtcSenderThread::send_and_free_messages(SrsSharedPtrMessage** msgs, int bool is_video = msg->is_video(); bool is_audio = msg->is_audio(); - int nn_rtp_pkts = (int)msg->rtp_packets.size(); - for (int j = 0; j < nn_rtp_pkts; j++) { - SrsRtpSharedPacket* pkt = msg->rtp_packets[j]; - send_and_free_message(msg, is_video, is_audio, pkt, skt); - } + if (is_audio) { + // Package opus packets to RTP packets. + vector rtp_packets; - *pnn += msg->size; - *pnn_rtp_pkts += nn_rtp_pkts; + for (int i = 0; i < msg->nn_extra_payloads(); i++) { + SrsSample* sample = msg->extra_payloads() + i; + if ((err = packet_opus(msg, sample, rtp_packets)) != srs_success) { + srs_warn("packet opus err %s", srs_error_summary(err).c_str()); srs_error_reset(err); + } + } + + int nn_rtp_pkts = (int)rtp_packets.size(); + for (int j = 0; j < nn_rtp_pkts; j++) { + SrsRtpSharedPacket* pkt = rtp_packets[j]; + send_and_free_message(msg, is_video, is_audio, pkt, skt); + } + + *pnn += msg->size; + *pnn_rtp_pkts += nn_rtp_pkts; + } else { + int nn_rtp_pkts = (int)msg->rtp_packets.size(); + for (int j = 0; j < nn_rtp_pkts; j++) { + SrsRtpSharedPacket* pkt = msg->rtp_packets[j]; + send_and_free_message(msg, is_video, is_audio, pkt, skt); + } + + *pnn += msg->size; + *pnn_rtp_pkts += nn_rtp_pkts; + } srs_freep(msg); } @@ -636,6 +662,24 @@ void SrsRtcSenderThread::send_and_free_message(SrsSharedPtrMessage* msg, bool is rtc_session->rtc_server->sendmmsg(skt->stfd(), mhdr); } +srs_error_t SrsRtcSenderThread::packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector& rtp_packets) +{ + srs_error_t err = srs_success; + + SrsRtpSharedPacket* packet = new SrsRtpSharedPacket(); + packet->rtp_header.set_marker(true); + if ((err = packet->create(timestamp, sequence++, kAudioSSRC, kOpusPayloadType, sample->bytes, sample->size)) != srs_success) { + return srs_error_wrap(err, "rtp packet encode"); + } + + // TODO: FIXME: Why 960? Need Refactoring? + timestamp += 960; + + rtp_packets.push_back(packet); + + return err; +} + SrsRtcSession::SrsRtcSession(SrsRtcServer* rtc_svr, const SrsRequest& req, const std::string& un, int context_id) { rtc_server = rtc_svr; diff --git a/trunk/src/app/srs_app_rtc_conn.hpp b/trunk/src/app/srs_app_rtc_conn.hpp index f448778b9..7e8019f54 100644 --- a/trunk/src/app/srs_app_rtc_conn.hpp +++ b/trunk/src/app/srs_app_rtc_conn.hpp @@ -126,6 +126,10 @@ private: uint32_t audio_ssrc; uint16_t video_payload_type; uint16_t audio_payload_type; +private: + // TODO: FIXME: How to handle timestamp overflow? + uint32_t timestamp; + uint16_t sequence; public: SrsUdpMuxSocket* sendonly_ukt; public: @@ -146,6 +150,8 @@ public: private: void send_and_free_messages(SrsSharedPtrMessage** msgs, int nb_msgs, SrsUdpMuxSocket* skt, int* pnn, int* pnn_rtp_pkts); void send_and_free_message(SrsSharedPtrMessage* msg, bool is_video, bool is_audio, SrsRtpSharedPacket* pkt, SrsUdpMuxSocket* skt); +private: + srs_error_t packet_opus(SrsSharedPtrMessage* shared_frame, SrsSample* sample, std::vector& rtp_packets); }; class SrsRtcSession diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 14e486835..49447b5de 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -538,10 +538,10 @@ public: bool bframe; public: SrsSample(); - virtual ~SrsSample(); + ~SrsSample(); public: // If we need to know whether sample is bframe, we have to parse the NALU payload. - virtual srs_error_t parse_bframe(); + srs_error_t parse_bframe(); }; /** diff --git a/trunk/src/kernel/srs_kernel_flv.cpp b/trunk/src/kernel/srs_kernel_flv.cpp index f74df7968..b0ced2f92 100644 --- a/trunk/src/kernel/srs_kernel_flv.cpp +++ b/trunk/src/kernel/srs_kernel_flv.cpp @@ -230,7 +230,8 @@ SrsSharedPtrMessage::SrsSharedPtrPayload::~SrsSharedPtrPayload() SrsSample* p = extra_payloads + i; srs_freep(p->bytes); } - srs_freep(extra_payloads); + srs_freepa(extra_payloads); + nn_extra_payloads = 0; } SrsSharedPtrMessage::SrsSharedPtrMessage() : timestamp(0), stream_id(0), size(0), payload(NULL) @@ -385,6 +386,17 @@ void SrsSharedPtrMessage::set_rtp_packets(const std::vector { rtp_packets = pkts; } + +void SrsSharedPtrMessage::set_extra_payloads(SrsSample* payloads, int nn_payloads) +{ + srs_assert(nn_payloads); + srs_assert(!ptr->extra_payloads); + + ptr->nn_extra_payloads = nn_payloads; + + ptr->extra_payloads = new SrsSample[nn_payloads]; + memcpy(ptr->extra_payloads, payloads, nn_payloads * sizeof(SrsSample)); +} #endif SrsFlvTransmuxer::SrsFlvTransmuxer() diff --git a/trunk/src/kernel/srs_kernel_flv.hpp b/trunk/src/kernel/srs_kernel_flv.hpp index b948d2596..ca5f24f09 100644 --- a/trunk/src/kernel/srs_kernel_flv.hpp +++ b/trunk/src/kernel/srs_kernel_flv.hpp @@ -358,6 +358,12 @@ public: public: #ifdef SRS_AUTO_RTC virtual void set_rtp_packets(const std::vector& pkts); + // Set extra samples, for example, when we transcode an AAC audio packet to OPUS, + // we may get more than one OPUS packets, we set these OPUS packets in extra payloads. + void set_extra_payloads(SrsSample* payloads, int nn_payloads); + // Get the extra payloads and the number of it. + int nn_extra_payloads() { return ptr->nn_extra_payloads; } + SrsSample* extra_payloads() { return ptr->extra_payloads; } #endif };