AI: API: Add audio_frames and video_frames to HTTP API. v7.0.122 (#4559) (#4564)

This PR adds separate audio and video frame counting to the HTTP API
(`/api/v1/streams/`) for better stream observability. The API now
reports three frame fields:
- `frames` - Total frames (video + audio)
- `video_frames` - Video frames/packets only
- `audio_frames` - Audio frames/packets only

This enhancement provides better visibility into stream composition and
helps detect issues with CBR/VBR streams, audio/video sync problems, and
codec-specific behavior.

**Before:**
```json
{
  "streams": [
    {
      "frames": 0, // video frames.
    }
  ]
}
```

**After:**
```json
{
  "streams": [
    {
      "frames": 6912, // video frames.
      "audio_frames": 5678, // audio frames.
      "video_frames": 1234, // video frames.
    }
  ]
}
```

Frame Counting Strategy
- All protocols report frames every N frames to balance accuracy and
performance
- Frames are counted at the protocol-specific message/packet level:
  - RTMP: Counts RTMP messages (video/audio)
  - WebRTC: Counts RTP packets (video/audio)
  - SRT: Counts MPEG-TS messages (H.264/HEVC/AAC)
This commit is contained in:
OSSRS-AI 2025-11-07 22:32:26 -05:00 committed by GitHub
parent f392f9a5a7
commit 1a96abc880
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
28 changed files with 214 additions and 28 deletions

View File

@ -7,6 +7,7 @@ The changelog for SRS.
<a name="v7-changes"></a>
## SRS 7.0 Changelog
* v7.0, 2025-11-08, AI: API: Add audio_frames and video_frames to HTTP API. v7.0.122 (#4559)
* v7.0, 2025-11-07, AI: WHIP: Return detailed HTTP error responses with proper status codes. v7.0.121 (#4502)
* v7.0, 2025-11-07, AI: HLS: Support query string in hls_key_url for JWT tokens. v7.0.120 (#4426)
* v7.0, 2025-11-07, AI: RTC: Support keep_original_ssrc to preserve SSRC and timestamps. v7.0.119 (#3850)

View File

@ -299,6 +299,7 @@ SrsPublishRecvThread::SrsPublishRecvThread(ISrsRtmpServer *rtmp_sdk, ISrsRequest
recv_error_ = srs_success;
nb_msgs_ = 0;
video_frames_ = 0;
audio_frames_ = 0;
error_ = srs_cond_new();
req_ = _req;
@ -349,6 +350,11 @@ uint64_t SrsPublishRecvThread::nb_video_frames()
return video_frames_;
}
uint64_t SrsPublishRecvThread::nb_audio_frames()
{
return audio_frames_;
}
srs_error_t SrsPublishRecvThread::error_code()
{
return srs_error_copy(recv_error_);
@ -396,6 +402,8 @@ srs_error_t SrsPublishRecvThread::consume(SrsRtmpCommonMessage *msg)
if (msg->header_.is_video()) {
video_frames_++;
} else if (msg->header_.is_audio()) {
audio_frames_++;
}
// log to show the time of recv thread.

View File

@ -190,6 +190,8 @@ SRS_DECLARE_PRIVATE: // clang-format on
int64_t nb_msgs_;
// The video frames we got.
uint64_t video_frames_;
// The audio frames we got.
uint64_t audio_frames_;
// For mr(merged read),
bool mr_;
int mr_fd_;
@ -219,6 +221,7 @@ public:
virtual srs_error_t wait(srs_utime_t tm);
virtual int64_t nb_msgs();
virtual uint64_t nb_video_frames();
virtual uint64_t nb_audio_frames();
virtual srs_error_t error_code();
virtual void set_cid(SrsContextId v);
virtual SrsContextId get_cid();

View File

@ -1211,7 +1211,7 @@ SrsRtcPublishStream::SrsRtcPublishStream(ISrsExecRtcAsyncTask *exec, ISrsExpire
pt_to_drop_ = 0;
nn_audio_frames_ = 0;
nn_rtp_pkts_ = 0;
nn_video_frames_ = 0;
format_ = new SrsRtcFormat();
twcc_enabled_ = false;
twcc_id_ = 0;
@ -1714,18 +1714,28 @@ void SrsRtcPublishStream::update_rtp_packet_stats(bool is_audio)
srs_error_t err = srs_success;
// Count RTP packets for statistics.
++nn_rtp_pkts_;
if (is_audio) {
++nn_audio_frames_;
} else {
++nn_video_frames_;
}
// Update the stat for frames, counting RTP packets as frames.
if (nn_rtp_pkts_ > 288) {
if ((err = stat_->on_video_frames(req_, (int)nn_rtp_pkts_)) != srs_success) {
srs_warn("RTC: stat frames err %s", srs_error_desc(err).c_str());
// Update the stat for video frames, counting RTP packets as frames.
if (nn_video_frames_ > 288) {
if ((err = stat_->on_video_frames(req_, nn_video_frames_)) != srs_success) {
srs_warn("RTC: stat video frames err %s", srs_error_desc(err).c_str());
srs_freep(err);
}
nn_rtp_pkts_ = 0;
nn_video_frames_ = 0;
}
// Update the stat for audio frames periodically.
if (nn_audio_frames_ > 288) {
if ((err = stat_->on_audio_frames(req_, nn_audio_frames_)) != srs_success) {
srs_warn("RTC: stat audio frames err %s", srs_error_desc(err).c_str());
srs_freep(err);
}
nn_audio_frames_ = 0;
}
}

View File

@ -542,8 +542,8 @@ SRS_DECLARE_PRIVATE: // clang-format on
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
SrsContextId cid_;
uint64_t nn_audio_frames_;
int nn_rtp_pkts_;
int nn_audio_frames_;
int nn_video_frames_;
ISrsRtcFormat *format_;
ISrsRtcPliWorker *pli_worker_;
SrsErrorPithyPrint *twcc_epp_;

View File

@ -988,6 +988,7 @@ srs_error_t SrsRtmpConn::do_publishing(SrsSharedPtr<SrsLiveSource> source, SrsPu
int64_t nb_msgs = 0;
uint64_t nb_frames = 0;
uint64_t nb_audio_frames = 0;
while (true) {
if ((err = trd_->pull()) != srs_success) {
return srs_error_wrap(err, "rtmp: thread quit");
@ -1029,6 +1030,12 @@ srs_error_t SrsRtmpConn::do_publishing(SrsSharedPtr<SrsLiveSource> source, SrsPu
}
nb_frames = rtrd->nb_video_frames();
// Update the stat for audio frames.
if ((err = stat_->on_audio_frames(req, (int)(rtrd->nb_audio_frames() - nb_audio_frames))) != srs_success) {
return srs_error_wrap(err, "rtmp: stat audio frames");
}
nb_audio_frames = rtrd->nb_audio_frames();
// reportable
if (pprint->can_print()) {
kbps_->sample();

View File

@ -554,7 +554,6 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
SrsUniquePtr<SrsPithyPrint> pprint(SrsPithyPrint::create_srt_publish());
int nb_packets = 0;
int nb_frames = 0;
// Max udp packet size equal to 1500.
char buf[1500];
@ -586,15 +585,6 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
}
++nb_packets;
++nb_frames;
// Update the stat for frames every 100 packets, counting SRT packets as frames.
if (nb_frames > 288) {
if ((err = stat_->on_video_frames(req_, nb_frames)) != srs_success) {
return srs_error_wrap(err, "srt: stat frames");
}
nb_frames = 0;
}
if ((err = on_srt_packet(buf, nb)) != srs_success) {
return srs_error_wrap(err, "srt: process packet");

View File

@ -941,6 +941,8 @@ SrsSrtFormat::SrsSrtFormat()
format_ = new SrsRtmpFormat();
video_codec_reported_ = false;
audio_codec_reported_ = false;
nn_video_frames_ = 0;
nn_audio_frames_ = 0;
stat_ = _srs_stat;
}
@ -988,12 +990,45 @@ srs_error_t SrsSrtFormat::on_srt_packet(SrsSrtPacket *pkt)
return err;
}
void SrsSrtFormat::update_ts_message_stats(bool is_audio)
{
srs_error_t err = srs_success;
// Count TS messages for statistics.
if (is_audio) {
++nn_audio_frames_;
} else {
++nn_video_frames_;
}
// Update the stat for video frames, counting TS messages as frames.
if (nn_video_frames_ > 288) {
if ((err = stat_->on_video_frames(req_, nn_video_frames_)) != srs_success) {
srs_warn("SRT: stat video frames err %s", srs_error_desc(err).c_str());
srs_freep(err);
}
nn_video_frames_ = 0;
}
// Update the stat for audio frames periodically.
if (nn_audio_frames_ > 288) {
if ((err = stat_->on_audio_frames(req_, nn_audio_frames_)) != srs_success) {
srs_warn("SRT: stat audio frames err %s", srs_error_desc(err).c_str());
srs_freep(err);
}
nn_audio_frames_ = 0;
}
}
srs_error_t SrsSrtFormat::on_ts_message(SrsTsMessage *msg)
{
srs_error_t err = srs_success;
// Only parse video and audio messages
if (msg->channel_->stream_ == SrsTsStreamVideoH264 || msg->channel_->stream_ == SrsTsStreamVideoHEVC) {
// Update statistics for video frames
update_ts_message_stats(false);
if (video_codec_reported_) {
return err;
}
@ -1016,6 +1051,9 @@ srs_error_t SrsSrtFormat::on_ts_message(SrsTsMessage *msg)
c->width_, c->height_);
}
} else if (msg->channel_->stream_ == SrsTsStreamAudioAAC) {
// Update statistics for audio frames
update_ts_message_stats(true);
if (audio_codec_reported_) {
return err;
}

View File

@ -179,6 +179,7 @@ public:
// clang-format off
SRS_DECLARE_PRIVATE: // clang-format on
void update_ts_message_stats(bool is_audio);
srs_error_t parse_video_codec(SrsTsMessage *msg);
srs_error_t parse_audio_codec(SrsTsMessage *msg);
@ -191,6 +192,9 @@ SRS_DECLARE_PRIVATE: // clang-format on
// Track whether we've already reported codec info to avoid duplicate updates
bool video_codec_reported_;
bool audio_codec_reported_;
// Frame counters for statistics reporting
int nn_video_frames_;
int nn_audio_frames_;
};
// Collect and build SRT TS packet to AV frames.

View File

@ -97,13 +97,15 @@ SrsStatisticStream::SrsStatisticStream()
kbps_ = new SrsKbps();
nb_clients_ = 0;
frames_ = new SrsPps();
video_frames_ = new SrsPps();
audio_frames_ = new SrsPps();
}
SrsStatisticStream::~SrsStatisticStream()
{
srs_freep(kbps_);
srs_freep(frames_);
srs_freep(video_frames_);
srs_freep(audio_frames_);
}
srs_error_t SrsStatisticStream::dumps(SrsJsonObject *obj)
@ -118,7 +120,9 @@ srs_error_t SrsStatisticStream::dumps(SrsJsonObject *obj)
obj->set("url", SrsJsonAny::str(url_.c_str()));
obj->set("live_ms", SrsJsonAny::integer(srsu2ms(srs_time_now_cached())));
obj->set("clients", SrsJsonAny::integer(nb_clients_));
obj->set("frames", SrsJsonAny::integer(frames_->sugar_));
obj->set("frames", SrsJsonAny::integer(video_frames_->sugar_ + audio_frames_->sugar_));
obj->set("audio_frames", SrsJsonAny::integer(audio_frames_->sugar_));
obj->set("video_frames", SrsJsonAny::integer(video_frames_->sugar_));
obj->set("send_bytes", SrsJsonAny::integer(kbps_->get_send_bytes()));
obj->set("recv_bytes", SrsJsonAny::integer(kbps_->get_recv_bytes()));
@ -395,7 +399,19 @@ srs_error_t SrsStatistic::on_video_frames(ISrsRequest *req, int nb_frames)
SrsStatisticVhost *vhost = create_vhost(req);
SrsStatisticStream *stream = create_stream(vhost, req);
stream->frames_->sugar_ += nb_frames;
stream->video_frames_->sugar_ += nb_frames;
return err;
}
srs_error_t SrsStatistic::on_audio_frames(ISrsRequest *req, int nb_frames)
{
srs_error_t err = srs_success;
SrsStatisticVhost *vhost = create_vhost(req);
SrsStatisticStream *stream = create_stream(vhost, req);
stream->audio_frames_->sugar_ += nb_frames;
return err;
}
@ -542,7 +558,8 @@ void SrsStatistic::kbps_sample()
for (it = streams_.begin(); it != streams_.end(); it++) {
SrsStatisticStream *stream = it->second;
stream->kbps_->sample();
stream->frames_->update();
stream->video_frames_->update();
stream->audio_frames_->update();
}
}
if (true) {

View File

@ -63,8 +63,10 @@ public:
public:
// The stream total kbps.
SrsKbps *kbps_;
// The fps of stream.
SrsPps *frames_;
// The fps of stream (video frames/packets).
SrsPps *video_frames_;
// The fps of audio (audio frames/packets).
SrsPps *audio_frames_;
public:
bool has_video_;
@ -148,6 +150,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta) = 0;
virtual void kbps_sample() = 0;
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames) = 0;
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames) = 0;
public:
// Get the server id, used to identify the server.
@ -240,6 +243,9 @@ public:
// When got videos, update the frames.
// We only stat the total number of video frames.
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
// When got audios, update the audio frames.
// We only stat the total number of audio frames.
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
// When publish stream.
// @param req the request object of publish connection.
// @param publisher_id The id of publish connection.

View File

@ -9,6 +9,6 @@
#define VERSION_MAJOR 7
#define VERSION_MINOR 0
#define VERSION_REVISION 121
#define VERSION_REVISION 122
#endif

View File

@ -1963,6 +1963,11 @@ srs_error_t MockStatisticForOriginHub::on_video_frames(ISrsRequest *req, int nb_
return srs_success;
}
srs_error_t MockStatisticForOriginHub::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockStatisticForOriginHub::server_id()
{
return "mock_server_id";

View File

@ -211,6 +211,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();

View File

@ -809,6 +809,11 @@ srs_error_t MockStatisticForResampleKbps::on_video_frames(ISrsRequest *req, int
return srs_success;
}
srs_error_t MockStatisticForResampleKbps::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockStatisticForResampleKbps::server_id()
{
return "mock_server_id";

View File

@ -260,6 +260,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();

View File

@ -827,6 +827,11 @@ srs_error_t MockStatisticForLiveStream::on_video_frames(ISrsRequest *req, int nb
return srs_success;
}
srs_error_t MockStatisticForLiveStream::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockStatisticForLiveStream::server_id()
{
return "mock_server_id";

View File

@ -168,6 +168,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();

View File

@ -1707,6 +1707,11 @@ srs_error_t MockStatisticForRtcApi::on_video_frames(ISrsRequest *req, int nb_fra
return srs_success;
}
srs_error_t MockStatisticForRtcApi::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockStatisticForRtcApi::server_id()
{
return server_id_;
@ -3189,7 +3194,7 @@ VOID TEST(StatisticTest, StreamMediaInfo)
// Verify frame count was accumulated correctly
stream = stat->find_stream_by_url(req->get_stream_url());
EXPECT_TRUE(stream != NULL);
EXPECT_EQ(55, stream->frames_->sugar_);
EXPECT_EQ(55, stream->video_frames_->sugar_);
}
// Test SrsStatistic audio sample rate handling for AAC 48000 Hz
@ -3806,6 +3811,11 @@ srs_error_t MockStatisticForHooks::on_video_frames(ISrsRequest *req, int nb_fram
return srs_success;
}
srs_error_t MockStatisticForHooks::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockStatisticForHooks::server_id()
{
return server_id_;

View File

@ -317,6 +317,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();
@ -566,6 +567,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();

View File

@ -2234,6 +2234,16 @@ void MockStatisticForHttpxConn::kbps_sample()
{
}
srs_error_t MockStatisticForHttpxConn::on_video_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
srs_error_t MockStatisticForHttpxConn::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
srs_error_t MockStatisticForHttpxConn::dumps_vhosts(SrsJsonArray *arr)
{
return srs_success;
@ -2254,6 +2264,41 @@ srs_error_t MockStatisticForHttpxConn::dumps_metrics(int64_t &send_bytes, int64_
return srs_success;
}
std::string MockStatisticForHttpxConn::server_id()
{
return "mock_server_id";
}
std::string MockStatisticForHttpxConn::service_id()
{
return "mock_service_id";
}
std::string MockStatisticForHttpxConn::service_pid()
{
return "mock_pid";
}
SrsStatisticVhost *MockStatisticForHttpxConn::find_vhost_by_id(std::string vid)
{
return NULL;
}
SrsStatisticStream *MockStatisticForHttpxConn::find_stream(std::string sid)
{
return NULL;
}
SrsStatisticStream *MockStatisticForHttpxConn::find_stream_by_url(std::string url)
{
return NULL;
}
SrsStatisticClient *MockStatisticForHttpxConn::find_client(std::string client_id)
{
return NULL;
}
void MockStatisticForHttpxConn::reset()
{
on_disconnect_called_ = false;

View File

@ -502,10 +502,19 @@ public:
virtual void on_stream_close(ISrsRequest *req);
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t dumps_vhosts(SrsJsonArray *arr);
virtual srs_error_t dumps_streams(SrsJsonArray *arr, int start, int count);
virtual srs_error_t dumps_clients(SrsJsonArray *arr, int start, int count);
virtual srs_error_t dumps_metrics(int64_t &send_bytes, int64_t &recv_bytes, int64_t &nstreams, int64_t &nclients, int64_t &total_nclients, int64_t &nerrs);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();
virtual SrsStatisticVhost *find_vhost_by_id(std::string vid);
virtual SrsStatisticStream *find_stream(std::string sid);
virtual SrsStatisticStream *find_stream_by_url(std::string url);
virtual SrsStatisticClient *find_client(std::string client_id);
void reset();
};

View File

@ -1257,6 +1257,11 @@ srs_error_t MockSrtStatistic::on_video_frames(ISrsRequest *req, int nb_frames)
return srs_success;
}
srs_error_t MockSrtStatistic::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockSrtStatistic::server_id()
{
return "mock_server_id";

View File

@ -57,6 +57,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();

View File

@ -1482,6 +1482,11 @@ srs_error_t MockStatisticForRtspPlayStream::on_video_frames(ISrsRequest *req, in
return srs_success;
}
srs_error_t MockStatisticForRtspPlayStream::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockStatisticForRtspPlayStream::server_id()
{
return "mock_server_id";

View File

@ -340,6 +340,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();

View File

@ -438,6 +438,11 @@ srs_error_t MockAppStatistic::on_video_frames(ISrsRequest *req, int nb_frames)
return srs_success;
}
srs_error_t MockAppStatistic::on_audio_frames(ISrsRequest *req, int nb_frames)
{
return srs_success;
}
std::string MockAppStatistic::server_id()
{
return "";

View File

@ -215,6 +215,7 @@ public:
virtual void kbps_add_delta(std::string id, ISrsKbpsDelta *delta);
virtual void kbps_sample();
virtual srs_error_t on_video_frames(ISrsRequest *req, int nb_frames);
virtual srs_error_t on_audio_frames(ISrsRequest *req, int nb_frames);
virtual std::string server_id();
virtual std::string service_id();
virtual std::string service_pid();