srs/trunk/src/app/srs_app_rtmp_conn.hpp
Haibo Chen(陈海博) ef048b0d65
RTC: Fix DVR missing first 4-6 seconds by initializing rate from SDP (#4541)
for issue #4418, #4151, #4076 .DVR Missing First Few Seconds of
Audio/Video

### Root Cause
When recording WebRTC streams to FLV files using DVR, the first 4-6
seconds of audio/video are missing. This occurs because:

1. **Packets are discarded before A/V sync is available**: The
RTC-to-RTMP conversion pipeline actively discards all RTP packets when
avsync_time <= 0.
2. **Original algorithm requires 2 RTCP SR packets**: The previous
implementation needed to receive two RTCP Sender Report (SR) packets
before it could calculate the rate for audio/video synchronization
timestamp conversion.
3. **Delay causes packet loss**: Since RTCP SR packets typically arrive
every 2-3 seconds, waiting for 2 SRs means 4-6 seconds of packets are
discarded before A/V sync becomes available.
4. **Audio SR arrives slower than video SR**: As reported in the issue,
video RTCP SR packets arrive much faster than audio SR packets. This
asymmetry causes audio packets to be discarded for a longer period,
resulting in the audio loss observed in DVR recordings.

### Solution
1. **Initialize rate from SDP**: Use the sample rate from SDP (Session
Description Protocol) to calculate the initial rate immediately when the
track is created.
Audio (Opus): 48000 Hz → rate = 48 (RTP units per millisecond)
Video (H.264/H.265): 90000 Hz → rate = 90 (RTP units per millisecond)
2. **Enable immediate A/V sync:** With the SDP rate available,
cal_avsync_time() can calculate valid timestamps from the very first RTP
packet, eliminating packet loss.
3. **Smooth transition to precise rate**: After receiving the 2nd RTCP
SR, update to the precisely calculated rate based on actual RTP/NTP
timestamp mapping.

## Configuration

Added new configuration option `init_rate_from_sdp` in the RTC vhost
section:

```nginx
vhost rtc.vhost.srs.com {
    rtc {
        # Whether initialize RTP rate from SDP sample rate for immediate A/V sync.
        # When enabled, the RTP rate (units per millisecond) is initialized from the SDP
        # sample rate (e.g., 90 for video 90kHz, 48 for audio 48kHz) before receiving
        # 2 RTCP SR packets. This allows immediate audio/video synchronization.
        # The rate will be updated to a more precise value after receiving the 2nd SR.
        # Overwrite by env SRS_VHOST_RTC_INIT_RATE_FROM_SDP for all vhosts.
        # Default: off
        init_rate_from_sdp off;
    }
}
```

**⚠️ Important Note**: This config defaults to **off** because:
-  When **enabled**: Fixes the audio loss problem (no missing first 4-6
seconds)
-  When **enabled**: VLC on macOS cannot play the video properly
-  Other platforms work fine (Windows, Linux)
-  FFplay works fine on all platforms

Users experiencing audio loss in DVR recordings can enable this option
if they don't need VLC macOS compatibility. We're investigating the VLC
macOS issue to make this feature safe to enable by default in the
future.

---------

Co-authored-by: winlin <winlinvip@gmail.com>
Co-authored-by: OSSRS-AI <winlinam@gmail.com>
2025-10-28 09:33:40 -04:00

316 lines
9.7 KiB
C++

//
// Copyright (c) 2013-2025 The SRS Authors
//
// SPDX-License-Identifier: MIT
//
#ifndef SRS_APP_RTMP_CONN_HPP
#define SRS_APP_RTMP_CONN_HPP
#include <srs_core.hpp>
#include <string>
#include <srs_app_reload.hpp>
#include <srs_app_st.hpp>
#include <srs_core_autofree.hpp>
#include <srs_protocol_conn.hpp>
#include <srs_protocol_rtmp_conn.hpp>
#include <srs_protocol_rtmp_stack.hpp>
class SrsServer;
class SrsRtmpServer;
class ISrsRequest;
class SrsResponse;
class SrsLiveSource;
class SrsRefer;
class SrsLiveConsumer;
class SrsRtmpCommonMessage;
class SrsStSocket;
class SrsHttpHooks;
class SrsBandwidth;
class SrsKbps;
class SrsRtmpClient;
class SrsMediaPacket;
class SrsQueueRecvThread;
class SrsPublishRecvThread;
class SrsSecurity;
class ISrsWakable;
class SrsRtmpCommonMessage;
class SrsRtmpCommand;
class SrsNetworkDelta;
class ISrsNetworkDelta;
class ISrsAppConfig;
class SrsSslConnection;
class ISrsResourceManager;
class ISrsStreamPublishTokenManager;
class ISrsLiveSourceManager;
class ISrsStatistic;
class ISrsHttpHooks;
class ISrsAppFactory;
class ISrsRtcSourceManager;
class ISrsSrtSourceManager;
class ISrsRtspSourceManager;
class ISrsRtmpServer;
class ISrsRtmpTransport;
class ISrsSecurity;
// The simple rtmp client for SRS.
class SrsSimpleRtmpClient : public SrsBasicRtmpClient
{
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
ISrsAppConfig *config_;
public:
SrsSimpleRtmpClient(std::string u, srs_utime_t ctm, srs_utime_t stm);
virtual ~SrsSimpleRtmpClient();
// clang-format off
SRS_DECLARE_PROTECTED: // clang-format on
virtual srs_error_t connect_app();
};
// Some information of client.
class SrsClientInfo
{
public:
// The type of client, play or publish.
SrsRtmpConnType type_;
// Whether the client connected at the edge server.
bool edge_;
// Original request object from client.
ISrsRequest *req_;
// Response object to client.
SrsResponse *res_;
public:
SrsClientInfo();
virtual ~SrsClientInfo();
};
// The transport layer for RTMP connections.
class ISrsRtmpTransport
{
public:
ISrsRtmpTransport();
virtual ~ISrsRtmpTransport();
public:
virtual srs_netfd_t fd() = 0;
virtual int osfd() = 0;
virtual ISrsProtocolReadWriter *io() = 0;
virtual srs_error_t handshake() = 0;
virtual const char *transport_type() = 0;
virtual srs_error_t set_socket_buffer(srs_utime_t buffer_v) = 0;
virtual srs_error_t set_tcp_nodelay(bool v) = 0;
virtual int64_t get_recv_bytes() = 0;
virtual int64_t get_send_bytes() = 0;
};
// The base transport layer for RTMP connections over plain TCP.
class SrsRtmpTransport : public ISrsRtmpTransport
{
// clang-format off
SRS_DECLARE_PROTECTED: // clang-format on
srs_netfd_t stfd_;
SrsTcpConnection *skt_;
public:
SrsRtmpTransport(srs_netfd_t c);
virtual ~SrsRtmpTransport();
public:
// Get the file descriptor for logging and identification
virtual srs_netfd_t fd();
virtual int osfd();
// Get the appropriate I/O interface (TCP)
virtual ISrsProtocolReadWriter *io();
// Perform handshake (no-op for plain RTMP)
virtual srs_error_t handshake();
// Get transport type description for logging
virtual const char *transport_type();
// Set socket buffer size
virtual srs_error_t set_socket_buffer(srs_utime_t buffer_v);
// Set TCP nodelay option
virtual srs_error_t set_tcp_nodelay(bool v);
// Get network statistics
virtual int64_t get_recv_bytes();
virtual int64_t get_send_bytes();
};
// The SSL/TLS transport layer for RTMPS connections.
class SrsRtmpsTransport : public SrsRtmpTransport
{
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
ISrsAppConfig *config_;
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
SrsSslConnection *ssl_;
public:
SrsRtmpsTransport(srs_netfd_t c);
virtual ~SrsRtmpsTransport();
public:
// Get the appropriate I/O interface (SSL)
virtual ISrsProtocolReadWriter *io();
// Perform SSL handshake
virtual srs_error_t handshake();
// Get transport type description for logging
virtual const char *transport_type();
};
// The RTMP connection, for client to publish or play stream.
class SrsRtmpConn : public ISrsConnection, // It's a resource.
public ISrsStartable,
public ISrsReloadHandler,
public ISrsCoroutineHandler,
public ISrsExpire
{
// For the thread to directly access any field of connection.
friend class SrsPublishRecvThread;
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
ISrsAppFactory *app_factory_;
ISrsResourceManager *manager_;
ISrsAppConfig *config_;
ISrsStreamPublishTokenManager *stream_publish_tokens_;
ISrsLiveSourceManager *live_sources_;
ISrsStatistic *stat_;
ISrsHttpHooks *hooks_;
ISrsRtcSourceManager *rtc_sources_;
ISrsSrtSourceManager *srt_sources_;
#ifdef SRS_RTSP
ISrsRtspSourceManager *rtsp_sources_;
#endif
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
ISrsRtmpServer *rtmp_;
SrsRefer *refer_;
SrsBandwidth *bandwidth_;
ISrsSecurity *security_;
// The wakable handler, maybe NULL.
// TODO: FIXME: Should refine the state for receiving thread.
ISrsWakable *wakable_;
// The elapsed duration in srs_utime_t
// For live play duration, for instance, rtmpdump to record.
srs_utime_t duration_;
// The MR(merged-write) sleep time in srs_utime_t.
srs_utime_t mw_sleep_;
int mw_msgs_;
// For realtime
// @see https://github.com/ossrs/srs/issues/257
bool realtime_;
// The minimal interval in srs_utime_t for delivery stream.
srs_utime_t send_min_interval_;
// The publish 1st packet timeout in srs_utime_t
srs_utime_t publish_1stpkt_timeout_;
// The publish normal packet timeout in srs_utime_t
srs_utime_t publish_normal_timeout_;
// Whether enable the tcp_nodelay.
bool tcp_nodelay_;
// About the rtmp client.
SrsClientInfo *info_;
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
ISrsRtmpTransport *transport_;
// Each connection start a green thread,
// when thread stop, the connection will be delete by server.
ISrsCoroutine *trd_;
// The ip and port of client.
std::string ip_;
int port_;
// The delta for statistic.
ISrsNetworkDelta *delta_;
SrsNetworkKbps *kbps_;
// The create time in milliseconds.
// for current connection to log self create time and calculate the living time.
int64_t create_time_;
public:
SrsRtmpConn(ISrsRtmpTransport *transport, std::string cip, int port);
void assemble();
virtual ~SrsRtmpConn();
// Interface ISrsResource.
public:
virtual std::string desc();
// clang-format off
SRS_DECLARE_PROTECTED: // clang-format on
virtual srs_error_t do_cycle();
public:
virtual ISrsKbpsDelta *delta();
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
// When valid and connected to vhost/app, service the client.
virtual srs_error_t service_cycle();
// The stream(play/publish) service cycle, identify client first.
virtual srs_error_t stream_service_cycle();
virtual srs_error_t check_vhost(bool try_default_vhost);
virtual srs_error_t playing(SrsSharedPtr<SrsLiveSource> source);
virtual srs_error_t do_playing(SrsSharedPtr<SrsLiveSource> source, SrsLiveConsumer *consumer, SrsQueueRecvThread *trd);
virtual srs_error_t publishing(SrsSharedPtr<SrsLiveSource> source);
virtual srs_error_t do_publishing(SrsSharedPtr<SrsLiveSource> source, SrsPublishRecvThread *trd);
virtual srs_error_t acquire_publish(SrsSharedPtr<SrsLiveSource> source);
virtual void release_publish(SrsSharedPtr<SrsLiveSource> source);
virtual srs_error_t handle_publish_message(SrsSharedPtr<SrsLiveSource> &source, SrsRtmpCommonMessage *msg);
virtual srs_error_t process_publish_message(SrsSharedPtr<SrsLiveSource> &source, SrsRtmpCommonMessage *msg);
virtual srs_error_t process_play_control_msg(SrsLiveConsumer *consumer, SrsRtmpCommonMessage *msg);
virtual void set_sock_options();
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
virtual srs_error_t check_edge_token_traverse_auth();
virtual srs_error_t do_token_traverse_auth(SrsRtmpClient *client);
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
// When the connection disconnect, call this method.
// e.g. log msg of connection and report to other system.
virtual srs_error_t on_disconnect();
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
virtual srs_error_t http_hooks_on_connect();
virtual void http_hooks_on_close();
virtual srs_error_t http_hooks_on_publish();
virtual void http_hooks_on_unpublish();
virtual srs_error_t http_hooks_on_play();
virtual void http_hooks_on_stop();
// Extract APIs from SrsTcpConnection.
// Interface ISrsStartable
public:
// Start the client green thread.
// when server get a client from listener,
// 1. server will create an concrete connection(for instance, RTMP connection),
// 2. then add connection to its connection manager,
// 3. start the client thread by invoke this start()
// when client cycle thread stop, invoke the on_thread_stop(), which will use server
// To remove the client by server->remove(this).
virtual srs_error_t start();
virtual void stop();
// Interface ISrsCoroutineHandler
public:
// The thread cycle function,
// when serve connection completed, terminate the loop which will terminate the thread,
// thread will invoke the on_thread_stop() when it terminated.
virtual srs_error_t cycle();
// Interface ISrsConnection.
public:
virtual std::string remote_ip();
virtual const SrsContextId &get_id();
// Interface ISrsExpire.
public:
virtual void expire();
};
#endif