AI: WebRTC: Report video/audio codec info and frame stats in HTTP API. v7.0.118 (#4554)

.vscode/launch.json

@@ -57,6 +57,34 @@
             "engineLogging": true
           }
         },
+        {
+          "name": "Launch SRS with console.conf",
+          "type": "cppdbg",
+          "request": "launch",
+          "program": "${workspaceFolder}/trunk/cmake/build/srs",
+          "args": ["-c", "console.conf"],
+          "stopAtEntry": false,
+          "cwd": "${workspaceFolder}/trunk",
+          "environment": [],
+          "externalConsole": false,
+          "linux": {
+            "MIMode": "gdb"
+          },
+          "osx": {
+            "MIMode": "lldb"
+          },
+          "setupCommands": [
+            {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+            }
+          ],
+          "preLaunchTask": "build",
+          "logging": {
+            "engineLogging": true
+          }
+        },
         {
           "name": "Launch srs-proxy",
           "type": "go",

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v7-changes"></a>
 
 ## SRS 7.0 Changelog
+* v7.0, 2025-11-05, AI: WebRTC: Report video/audio codec info and frame stats in HTTP API. v7.0.118 (#4554)
 * v7.0, 2025-11-04, AI: SRT: Report video/audio codec info and frame stats in HTTP API. v7.0.117 (#4554)
 * v7.0, 2025-11-03, Merge [#4556](https://github.com/ossrs/srs/pull/4556): Fill missing defs for H264/AVC video levels. v7.0.116 (#4556)
 * v7.0, 2025-10-31, Merge [#4547](https://github.com/ossrs/srs/pull/4547): Add ignore configuration for cursor. v7.0.115 (#4547)

trunk/src/app/srs_app_rtc_conn.cpp

@@ -1208,6 +1208,8 @@ SrsRtcPublishStream::SrsRtcPublishStream(ISrsExecRtcAsyncTask *exec, ISrsExpire
     pt_to_drop_ = 0;
 
     nn_audio_frames_ = 0;
+    nn_rtp_pkts_ = 0;
+    format_ = new SrsRtcFormat();
     twcc_enabled_ = false;
     twcc_id_ = 0;
     twcc_fb_count_ = 0;
@@ -1260,6 +1262,7 @@ SrsRtcPublishStream::~SrsRtcPublishStream()
     srs_freep(pli_worker_);
     srs_freep(twcc_epp_);
     srs_freep(pli_epp_);
+    srs_freep(format_);
     srs_freep(req_);
 
     // update the statistic when client coveried.
@@ -1284,6 +1287,10 @@ srs_error_t SrsRtcPublishStream::initialize(ISrsRequest *r, SrsRtcSourceDescript
 
     req_ = r->copy();
 
+    if ((err = format_->initialize(req_)) != srs_success) {
+        return srs_error_wrap(err, "initialize format");
+    }
+
     if ((err = timer_rtcp_->initialize()) != srs_success) {
         return srs_error_wrap(err, "initialize timer rtcp");
     }
@@ -1670,6 +1677,15 @@ srs_error_t SrsRtcPublishStream::do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuff
         return srs_error_new(ERROR_RTC_RTP, "unknown ssrc=%u", ssrc);
     }
 
+    // Report codec information to statistics on first RTP packet.
+    if ((err = format_->on_rtp_packet(track, is_audio)) != srs_success) {
+        srs_warn("RTC: format packet err %s", srs_error_desc(err).c_str());
+        srs_freep(err);
+    }
+
+    // Update RTP packet statistics.
+    update_rtp_packet_stats(is_audio);
+
     // Consume packet by track.
     if ((err = track->on_rtp(source_, pkt)) != srs_success) {
         return srs_error_wrap(err, "audio track, SSRC=%u, SEQ=%u", ssrc, pkt->header_.get_sequence());
@@ -1690,6 +1706,26 @@ srs_error_t SrsRtcPublishStream::do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuff
     return err;
 }
 
+void SrsRtcPublishStream::update_rtp_packet_stats(bool is_audio)
+{
+    srs_error_t err = srs_success;
+
+    // Count RTP packets for statistics.
+    ++nn_rtp_pkts_;
+    if (is_audio) {
+        ++nn_audio_frames_;
+    }
+
+    // Update the stat for frames, counting RTP packets as frames.
+    if (nn_rtp_pkts_ > 288) {
+        if ((err = stat_->on_video_frames(req_, (int)nn_rtp_pkts_)) != srs_success) {
+            srs_warn("RTC: stat frames err %s", srs_error_desc(err).c_str());
+            srs_freep(err);
+        }
+        nn_rtp_pkts_ = 0;
+    }
+}
+
 srs_error_t SrsRtcPublishStream::check_send_nacks()
 {
     srs_error_t err = srs_success;

trunk/src/app/srs_app_rtc_conn.hpp

@@ -74,6 +74,7 @@ class ISrsCoroutine;
 class ISrsDtlsCertificate;
 class SrsRtcRecvTrack;
 class ISrsRtcPlayStream;
+class ISrsRtcFormat;
 
 const uint8_t kSR = 200;
 const uint8_t kRR = 201;
@@ -542,6 +543,8 @@ SRS_DECLARE_PRIVATE: // clang-format on
 SRS_DECLARE_PRIVATE: // clang-format on
     SrsContextId cid_;
     uint64_t nn_audio_frames_;
+    int nn_rtp_pkts_;
+    ISrsRtcFormat *format_;
     ISrsRtcPliWorker *pli_worker_;
     SrsErrorPithyPrint *twcc_epp_;
 
@@ -619,6 +622,7 @@ public:
 // clang-format off
 SRS_DECLARE_PRIVATE: // clang-format on
     srs_error_t do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuffer *buf);
+    void update_rtp_packet_stats(bool is_audio);
 
 public:
     srs_error_t check_send_nacks();

trunk/src/app/srs_app_rtc_source.cpp

@@ -3279,6 +3279,11 @@ std::string SrsRtcRecvTrack::get_track_id()
     return track_desc_->id_;
 }
 
+SrsRtcTrackDescription *SrsRtcRecvTrack::get_track_desc()
+{
+    return track_desc_;
+}
+
 srs_error_t SrsRtcRecvTrack::on_nack(SrsRtpPacket **ppkt)
 {
     srs_error_t err = srs_success;
@@ -3774,3 +3779,147 @@ uint32_t SrsRtcSSRCGenerator::generate_ssrc()
 
     return ++ssrc_num_;
 }
+
+ISrsRtcFormat::ISrsRtcFormat()
+{
+}
+
+ISrsRtcFormat::~ISrsRtcFormat()
+{
+}
+
+SrsRtcFormat::SrsRtcFormat()
+{
+    req_ = NULL;
+    video_codec_reported_ = false;
+    audio_codec_reported_ = false;
+
+    stat_ = _srs_stat;
+}
+
+SrsRtcFormat::~SrsRtcFormat()
+{
+    req_ = NULL;
+
+    stat_ = NULL;
+}
+
+srs_error_t SrsRtcFormat::initialize(ISrsRequest *req)
+{
+    req_ = req;
+    return srs_success;
+}
+
+srs_error_t SrsRtcFormat::on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio)
+{
+    srs_error_t err = srs_success;
+
+    SrsRtcTrackDescription *track_desc = track ? track->get_track_desc() : NULL;
+    if (!req_ || !track_desc || !track_desc->media_) {
+        return err;
+    }
+
+    if (is_audio) {
+        // Only report once
+        if (audio_codec_reported_) {
+            return err;
+        }
+        audio_codec_reported_ = true;
+
+        SrsCodecPayload *media = track_desc->media_;
+        SrsAudioCodecId codec_id = (SrsAudioCodecId)media->codec(false);
+
+        // Parse channels and sample rate from track description
+        if (codec_id == SrsAudioCodecIdOpus) {
+            SrsAudioPayload *audio_media = dynamic_cast<SrsAudioPayload *>(media);
+            if (!audio_media) {
+                return err;
+            }
+
+            // Get sample rate from media payload
+            SrsAudioSampleRate sample_rate = (SrsAudioSampleRate)audio_media->sample_;
+
+            // Get channels from audio payload
+            SrsAudioChannels channels = (SrsAudioChannels)audio_media->channel_;
+
+            if ((err = stat_->on_audio_info(req_, codec_id, sample_rate, channels,
+                                             SrsAacObjectTypeReserved)) != srs_success) {
+                return srs_error_wrap(err, "stat audio info");
+            }
+            srs_trace("RTC: parsed %s codec, sample_rate=%dHz, channels=%d",
+                      srs_audio_codec_id2str(codec_id).c_str(), sample_rate, channels);
+        }
+    } else {
+        // Only report once
+        if (video_codec_reported_) {
+            return err;
+        }
+        video_codec_reported_ = true;
+
+        SrsCodecPayload *media = track_desc->media_;
+        SrsVideoCodecId codec_id = (SrsVideoCodecId)media->codec(true);
+
+        // Parse profile and level from track description
+        if (codec_id == SrsVideoCodecIdAVC) {
+            SrsVideoPayload *video_media = dynamic_cast<SrsVideoPayload *>(media);
+            if (!video_media || video_media->h264_param_.profile_level_id_.empty()) {
+                return err;
+            }
+
+            // Parse profile_level_id hex string (e.g., "42e01f")
+            // Format: PPCCLL where PP=profile_idc, CC=constraint_set, LL=level_idc
+            std::string profile_level_id = video_media->h264_param_.profile_level_id_;
+            if (profile_level_id.length() != 6) {
+                return err;
+            }
+
+            // Decode hex string to bytes
+            // srs_hex_decode_string expects size to be the hex string length (6 chars = 3 bytes)
+            uint8_t bytes[3];
+            int hex_len = (int)profile_level_id.length();
+            int r0 = srs_hex_decode_string(bytes, profile_level_id.c_str(), hex_len);
+            if (r0 != (int)sizeof(bytes)) {
+                srs_trace("RTC: failed to decode profile_level_id hex string: %s, r0=%d", profile_level_id.c_str(), r0);
+                video_codec_reported_ = true;
+                return err;
+            }
+
+            // Extract profile and level from the decoded bytes
+            SrsAvcProfile profile = (SrsAvcProfile)bytes[0];
+            SrsAvcLevel level = (SrsAvcLevel)bytes[2];
+
+            if ((err = stat_->on_video_info(req_, codec_id, profile, level, 0, 0)) != srs_success) {
+                return srs_error_wrap(err, "stat video info");
+            }
+            srs_trace("RTC: parsed %s codec, profile=%s, level=%s",
+                      srs_video_codec_id2str(codec_id).c_str(),
+                      srs_avc_profile2str(profile).c_str(),
+                      srs_avc_level2str(level).c_str());
+        } else if (codec_id == SrsVideoCodecIdHEVC) {
+            SrsVideoPayload *video_media = dynamic_cast<SrsVideoPayload *>(media);
+            if (!video_media || video_media->h265_param_.profile_id_.empty() ||
+                video_media->h265_param_.level_id_.empty()) {
+                return err;
+            }
+
+            // Parse HEVC profile_id and level_id from SDP parameters
+            // profile_id is a decimal string (e.g., "1" for Main profile)
+            // level_id is a decimal string (e.g., "93" for Level 3.1)
+            int profile_id = atoi(video_media->h265_param_.profile_id_.c_str());
+            int level_id = atoi(video_media->h265_param_.level_id_.c_str());
+
+            SrsHevcProfile profile = (SrsHevcProfile)profile_id;
+            SrsHevcLevel level = (SrsHevcLevel)level_id;
+
+            if ((err = stat_->on_video_info(req_, codec_id, profile, level, 0, 0)) != srs_success) {
+                return srs_error_wrap(err, "stat video info");
+            }
+            srs_trace("RTC: parsed %s codec, profile=%s, level=%d",
+                      srs_video_codec_id2str(codec_id).c_str(),
+                      srs_hevc_profile2str(profile).c_str(),
+                      level_id);
+        }
+    }
+
+    return err;
+}

trunk/src/app/srs_app_rtc_source.hpp

@@ -45,6 +45,7 @@ class ISrsRtcConsumer;
 class ISrsCircuitBreaker;
 class ISrsRtcPublishStream;
 class ISrsAppFactory;
+class ISrsStatistic;
 
 // Firefox defaults as 109, Chrome is 111.
 const int kAudioPayloadType = 111;
@@ -899,6 +900,7 @@ public:
     bool set_track_status(bool active);
     bool get_track_status();
     std::string get_track_id();
+    SrsRtcTrackDescription *get_track_desc();
 
 public:
     // Note that we can set the pkt to NULL to avoid copy, for example, if the NACK cache the pkt and
@@ -1144,4 +1146,37 @@ public:
     uint32_t generate_ssrc();
 };
 
+// The interface for RTC format.
+class ISrsRtcFormat
+{
+public:
+    ISrsRtcFormat();
+    virtual ~ISrsRtcFormat();
+
+public:
+    virtual srs_error_t initialize(ISrsRequest *req) = 0;
+    virtual srs_error_t on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio) = 0;
+};
+
+// Lightweight format parser for RTC streams to extract codec information
+// from RTP packets and update statistics.
+class SrsRtcFormat : public ISrsRtcFormat
+{
+public:
+    SrsRtcFormat();
+    virtual ~SrsRtcFormat();
+
+public:
+    virtual srs_error_t initialize(ISrsRequest *req);
+    virtual srs_error_t on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio);
+
+// clang-format off
+SRS_DECLARE_PRIVATE: // clang-format on
+    ISrsRequest *req_;
+    ISrsStatistic *stat_;
+    // Track whether we've already reported codec info to avoid duplicate updates
+    bool video_codec_reported_;
+    bool audio_codec_reported_;
+};
+
 #endif

trunk/src/app/srs_app_srt_conn.cpp

@@ -554,6 +554,7 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
     SrsUniquePtr<SrsPithyPrint> pprint(SrsPithyPrint::create_srt_publish());
 
     int nb_packets = 0;
+    int nb_frames = 0;
 
     // Max udp packet size equal to 1500.
     char buf[1500];
@@ -585,6 +586,15 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
         }
 
         ++nb_packets;
+        ++nb_frames;
+
+        // Update the stat for frames every 100 packets, counting SRT packets as frames.
+        if (nb_frames > 288) {
+            if ((err = stat_->on_video_frames(req_, nb_frames)) != srs_success) {
+                return srs_error_wrap(err, "srt: stat frames");
+            }
+            nb_frames = 0;
+        }
 
         if ((err = on_srt_packet(buf, nb)) != srs_success) {
             return srs_error_wrap(err, "srt: process packet");

trunk/src/core/srs_core_version7.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR 7
 #define VERSION_MINOR 0
-#define VERSION_REVISION 117
+#define VERSION_REVISION 118
 
 #endif

trunk/src/kernel/srs_kernel_utility.cpp

@@ -653,16 +653,19 @@ srs_error_t SrsPath::mkdir_all(string dir)
 }
 
 // fromHexChar converts a hex character into its value and a success flag.
-uint8_t srs_from_hex_char(uint8_t c)
+int srs_from_hex_char(uint8_t c, uint8_t *out)
 {
     if ('0' <= c && c <= '9') {
-        return c - '0';
+        *out = c - '0';
+        return 0;
     }
     if ('a' <= c && c <= 'f') {
-        return c - 'a' + 10;
+        *out = c - 'a' + 10;
+        return 0;
     }
     if ('A' <= c && c <= 'F') {
-        return c - 'A' + 10;
+        *out = c - 'A' + 10;
+        return 0;
     }
 
     return -1;
@@ -702,18 +705,18 @@ char *srs_hex_encode_to_string_lowercase(char *des, const u_int8_t *src, int len
 
 int srs_hex_decode_string(uint8_t *data, const char *p, int size)
 {
-    if (size <= 0 || (size % 2) == 1) {
+    if (!p || size <= 0 || (size % 2) == 1) {
         return -1;
     }
 
     for (int i = 0; i < (int)size / 2; i++) {
-        uint8_t a = srs_from_hex_char(p[i * 2]);
+        uint8_t a = 0;
-        if (a == (uint8_t)-1) {
+        if (srs_from_hex_char(p[i * 2], &a) == -1) {
             return -1;
         }
 
-        uint8_t b = srs_from_hex_char(p[i * 2 + 1]);
+        uint8_t b = 0;
-        if (b == (uint8_t)-1) {
+        if (srs_from_hex_char(p[i * 2 + 1], &b) == -1) {
             return -1;
         }
 

trunk/src/utest/srs_utest_ai24.cpp

@@ -1093,6 +1093,67 @@ VOID TEST(KernelUtilityTest, StringsDumpsHexWithString)
     EXPECT_TRUE(empty_result.empty());
 }
 
+// Test: srs_hex_decode_string decodes hex string to bytes
+VOID TEST(KernelUtilityTest, HexDecodeString)
+{
+    // Test normal case: decode valid hex string
+    if (true) {
+        std::string hex_str = "42e01f";
+        uint8_t data[3];
+
+        int result = srs_hex_decode_string(data, hex_str.c_str(), (int)hex_str.length());
+
+        EXPECT_EQ(3, result);
+        EXPECT_EQ(0x42, data[0]);
+        EXPECT_EQ(0xe0, data[1]);
+        EXPECT_EQ(0x1f, data[2]);
+    }
+
+    // Test uppercase hex string
+    if (true) {
+        std::string hex_str = "ABCDEF";
+        uint8_t data[3];
+
+        int result = srs_hex_decode_string(data, hex_str.c_str(), (int)hex_str.length());
+
+        EXPECT_EQ(3, result);
+        EXPECT_EQ(0xAB, data[0]);
+        EXPECT_EQ(0xCD, data[1]);
+        EXPECT_EQ(0xEF, data[2]);
+    }
+
+    // Test mixed case hex string
+    if (true) {
+        std::string hex_str = "aB12Cd";
+        uint8_t data[3];
+
+        int result = srs_hex_decode_string(data, hex_str.c_str(), (int)hex_str.length());
+
+        EXPECT_EQ(3, result);
+        EXPECT_EQ(0xAB, data[0]);
+        EXPECT_EQ(0x12, data[1]);
+        EXPECT_EQ(0xCD, data[2]);
+    }
+
+    // Test error case: NULL pointer
+    if (true) {
+        uint8_t data[3];
+        EXPECT_EQ(-1, srs_hex_decode_string(data, NULL, 6));
+    }
+
+    // Test error case: odd length (not pairs of hex digits)
+    if (true) {
+        uint8_t data[3];
+        EXPECT_EQ(-1, srs_hex_decode_string(data, "abc", 3));
+    }
+
+    // Test error case: invalid hex character
+    if (true) {
+        uint8_t data[3];
+        EXPECT_EQ(-1, srs_hex_decode_string(data, "abcg", 4));
+    }
+}
+
 // Test: srs_is_boolean checks if string is "true" or "false"
 VOID TEST(AppUtilityTest, IsBoolean)
 {

trunk/src/utest/srs_utest_manual_mock.cpp

@@ -547,6 +547,45 @@ void MockRtcPacketSender::set_send_packet_error(srs_error_t err)
     send_packet_error_ = err;
 }
 
+// MockRtcFormat implementation
+MockRtcFormat::MockRtcFormat()
+{
+    initialize_error_ = srs_success;
+    on_rtp_packet_error_ = srs_success;
+    initialize_count_ = 0;
+    on_rtp_packet_count_ = 0;
+}
+
+MockRtcFormat::~MockRtcFormat()
+{
+    srs_freep(initialize_error_);
+    srs_freep(on_rtp_packet_error_);
+}
+
+srs_error_t MockRtcFormat::initialize(ISrsRequest *req)
+{
+    initialize_count_++;
+    return srs_error_copy(initialize_error_);
+}
+
+srs_error_t MockRtcFormat::on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio)
+{
+    on_rtp_packet_count_++;
+    return srs_error_copy(on_rtp_packet_error_);
+}
+
+void MockRtcFormat::set_initialize_error(srs_error_t err)
+{
+    srs_freep(initialize_error_);
+    initialize_error_ = srs_error_copy(err);
+}
+
+void MockRtcFormat::set_on_rtp_packet_error(srs_error_t err)
+{
+    srs_freep(on_rtp_packet_error_);
+    on_rtp_packet_error_ = srs_error_copy(err);
+}
+
 // Mock RTC packet receiver implementation
 MockRtcPacketReceiver::MockRtcPacketReceiver()
 {

trunk/src/utest/srs_utest_manual_mock.hpp

@@ -263,6 +263,26 @@ public:
     void set_send_packet_error(srs_error_t err);
 };
 
+// Mock RTC format for testing
+class MockRtcFormat : public ISrsRtcFormat
+{
+public:
+    srs_error_t initialize_error_;
+    srs_error_t on_rtp_packet_error_;
+    int initialize_count_;
+    int on_rtp_packet_count_;
+
+public:
+    MockRtcFormat();
+    virtual ~MockRtcFormat();
+
+public:
+    virtual srs_error_t initialize(ISrsRequest *req);
+    virtual srs_error_t on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio);
+    void set_initialize_error(srs_error_t err);
+    void set_on_rtp_packet_error(srs_error_t err);
+};
+
 // Mock app config for testing
 class MockAppConfig : public ISrsAppConfig
 {

trunk/src/utest/srs_utest_workflow_rtc_publishstream.cpp

@@ -45,6 +45,7 @@ VOID TEST(BasicWorkflowRtcPublishStreamTest, ManuallyVerify)
     MockExpire mock_expire;
     MockRtcPacketReceiver mock_receiver;
     MockRtcTrackDescriptionFactory track_factory;
+    MockRtcFormat mock_format;
     SrsContextId cid;
     cid.set_value("test-publish-stream-cid");
 
@@ -57,6 +58,10 @@ VOID TEST(BasicWorkflowRtcPublishStreamTest, ManuallyVerify)
         publish_stream->config_ = &mock_config;
         publish_stream->rtc_sources_ = &mock_rtc_sources;
         publish_stream->stat_ = &mock_stat;
+
+        // Replace the real format_ with mock format
+        srs_freep(publish_stream->format_);
+        publish_stream->format_ = &mock_format;
     }
 
     // Create stream description with audio and video tracks
@@ -96,4 +101,8 @@ VOID TEST(BasicWorkflowRtcPublishStreamTest, ManuallyVerify)
 
     // Stop the publish stream
     publish_stream->stop();
+
+    // Before destroying publish_stream, set format_ to NULL to prevent double-free
+    // since mock_format is a stack variable
+    publish_stream->format_ = NULL;
 }