From 339897e0c761b462c5a0ab8123d9713403e37b25 Mon Sep 17 00:00:00 2001 From: Jacob Su Date: Tue, 12 Aug 2025 08:55:06 +0800 Subject: [PATCH] Feature: Support HLS with fmp4 segment for HEVC/LLHLS. v7.0.51 (#4159) Currently, SRS only supports HLS with MPEG-TS format segment files, but for LL-HLS and HEVC, it requires the fMP4 format. See #4327 for details. Furthermore, fMP4 has a smaller overhead compared to TS, and fMP4 can be used for DVR. In short, fMP4 is definitely the future segment format for HLS. Start SRS with the config file that enables HLS with fMP4: ``` ./objs/srs -c conf/hls.mp4.conf ``` Publish stream by FFmpeg: ``` ffmpeg -re -i doc/source.flv -c copy -f flv rtmp://localhost/live/livestream ``` Play the stream by SRS player: [http://localhost:8080/live/livestream.m3u8](http://localhost:8080/players/srs_player.html?stream=livestream.m3u8) Finished by AI: * [AI: Change init.mp4 to the same directory of m3u8.](https://github.com/ossrs/srs/pull/4159/commits/17621c8442f0c3d4c327eb8722c6ffc4efad73e3) * [AI: Fix the error handling bug.](https://github.com/ossrs/srs/pull/4159/commits/af3758a592e818c0aba36999da3d385d19293d23) * [AI: Fix Chrome stuttering problem.](https://github.com/ossrs/srs/pull/4159/commits/aaab60c314b584175c34ccd5b9dbd5ba3adfbb70) --------- Co-authored-by: winlin --- README.md | 5 - trunk/3rdparty/srs-bench/blackbox/hls_test.go | 83 ++ trunk/3rdparty/srs-bench/blackbox/util.go | 2 + trunk/conf/full.conf | 45 + trunk/conf/hls.mp4.conf | 29 + trunk/configure | 2 +- trunk/doc/CHANGELOG.md | 1 + trunk/src/app/srs_app_config.cpp | 65 +- trunk/src/app/srs_app_config.hpp | 7 + trunk/src/app/srs_app_hls.cpp | 1186 ++++++++++++++- trunk/src/app/srs_app_hls.hpp | 273 +++- trunk/src/app/srs_app_http_static.cpp | 29 +- trunk/src/app/srs_app_http_static.hpp | 1 + trunk/src/app/srs_app_server.hpp | 2 + trunk/src/core/srs_core_version7.hpp | 2 +- trunk/src/kernel/srs_kernel_codec.hpp | 1 + trunk/src/kernel/srs_kernel_mp4.cpp | 1281 ++++++++++++++++- trunk/src/kernel/srs_kernel_mp4.hpp | 469 +++++- .../src/protocol/srs_protocol_http_stack.cpp | 27 +- .../src/protocol/srs_protocol_http_stack.hpp | 3 +- trunk/src/utest/srs_utest_config.cpp | 26 +- trunk/src/utest/srs_utest_fmp4.cpp | 810 +++++++++++ trunk/src/utest/srs_utest_fmp4.hpp | 15 + trunk/src/utest/srs_utest_http.cpp | 44 + trunk/src/utest/srs_utest_mp4.cpp | 484 ++++++- 25 files changed, 4751 insertions(+), 141 deletions(-) create mode 100644 trunk/conf/hls.mp4.conf create mode 100644 trunk/src/utest/srs_utest_fmp4.cpp create mode 100644 trunk/src/utest/srs_utest_fmp4.hpp diff --git a/README.md b/README.md index 2348e00b8..ce2410922 100755 --- a/README.md +++ b/README.md @@ -3,14 +3,9 @@ ![](http://ossrs.net/gif/v1/sls.gif?site=github.com&path=/srs/develop) [![](https://github.com/ossrs/srs/actions/workflows/codeql-analysis.yml/badge.svg?branch=develop)](https://github.com/ossrs/srs/actions?query=workflow%3ACodeQL+branch%3Adevelop) [![](https://github.com/ossrs/srs/actions/workflows/release.yml/badge.svg)](https://github.com/ossrs/srs/actions/workflows/release.yml?query=workflow%3ARelease) -[![](https://github.com/ossrs/srs/actions/workflows/test.yml/badge.svg?branch=develop)](https://github.com/ossrs/srs/actions?query=workflow%3ATest+branch%3Adevelop) -[![](https://codecov.io/gh/ossrs/srs/branch/develop/graph/badge.svg?token=Zx2LhdtA39)](https://app.codecov.io/gh/ossrs/srs/tree/develop) -[![](https://ossrs.net/wiki/images/wechat-badge4.svg)](https://ossrs.net/lts/zh-cn/contact#discussion) [![](https://img.shields.io/twitter/follow/srs_server?style=social)](https://twitter.com/srs_server) [![](https://img.shields.io/badge/SRS-YouTube-red)](https://www.youtube.com/@srs_server) [![](https://badgen.net/discord/members/yZ4BnPmHAd)](https://discord.gg/yZ4BnPmHAd) -[![](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fossrs%2Fsrs.svg?type=small)](https://app.fossa.com/projects/git%2Bgithub.com%2Fossrs%2Fsrs?ref=badge_small) -[![](https://badgen.net/badge/srs/stackoverflow/orange?icon=terminal)](https://stackoverflow.com/questions/tagged/simple-realtime-server) [![](https://opencollective.com/srs-server/tiers/badge.svg)](https://opencollective.com/srs-server) [![](https://img.shields.io/docker/pulls/ossrs/srs)](https://hub.docker.com/r/ossrs/srs/tags) diff --git a/trunk/3rdparty/srs-bench/blackbox/hls_test.go b/trunk/3rdparty/srs-bench/blackbox/hls_test.go index 5b43e9a7d..395bf6da3 100644 --- a/trunk/3rdparty/srs-bench/blackbox/hls_test.go +++ b/trunk/3rdparty/srs-bench/blackbox/hls_test.go @@ -114,3 +114,86 @@ func TestFast_RtmpPublish_HlsPlay_Basic(t *testing.T) { } } } + +func TestFast_RtmpPublish_HlsPlay_Fmp4(t *testing.T) { + // This case is run in parallel. + t.Parallel() + + // Setup the max timeout for this case. + ctx, cancel := context.WithTimeout(logger.WithContext(context.Background()), time.Duration(*srsTimeout)*time.Millisecond) + defer cancel() + + // Check a set of errors. + var r0, r1, r2, r3, r4 error + defer func(ctx context.Context) { + if err := filterTestError(ctx.Err(), r0, r1, r2, r3, r4); err != nil { + t.Errorf("Fail for err %+v", err) + } else { + logger.Tf(ctx, "test done with err %+v", err) + } + }(ctx) + + var wg sync.WaitGroup + defer wg.Wait() + + // Start SRS server and wait for it to be ready. + svr := NewSRSServer(func(v *srsServer) { + v.envs = []string{ + "SRS_HTTP_SERVER_ENABLED=on", + "SRS_VHOST_HLS_ENABLED=on", + "SRS_VHOST_HLS_HLS_USE_FMP4=on", + } + }) + wg.Add(1) + go func() { + defer wg.Done() + r0 = svr.Run(ctx, cancel) + }() + + // Start FFmpeg to publish stream. + streamID := fmt.Sprintf("stream-%v-%v", os.Getpid(), rand.Int()) + streamURL := fmt.Sprintf("rtmp://localhost:%v/live/%v", svr.RTMPPort(), streamID) + ffmpeg := NewFFmpeg(func(v *ffmpegClient) { + v.args = []string{ + "-stream_loop", "-1", "-re", "-i", *srsPublishAvatar, "-c", "copy", "-f", "flv", streamURL, + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r1 = ffmpeg.Run(ctx, cancel) + }() + + // Start FFprobe to detect and verify stream. + duration := time.Duration(*srsFFprobeDuration) * time.Millisecond + ffprobe := NewFFprobe(func(v *ffprobeClient) { + v.dvrFile = path.Join(svr.WorkDir(), "objs", fmt.Sprintf("srs-ffprobe-%v.mp4", streamID)) + v.streamURL = fmt.Sprintf("http://localhost:%v/live/%v.m3u8", svr.HTTPPort(), streamID) + v.duration, v.timeout = duration, time.Duration(*srsFFprobeTimeout)*time.Millisecond + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r2 = ffprobe.Run(ctx, cancel) + }() + + // Fast quit for probe done. + select { + case <-ctx.Done(): + case <-ffprobe.ProbeDoneCtx().Done(): + defer cancel() + + str, m := ffprobe.Result() + if len(m.Streams) != 2 { + r3 = errors.Errorf("invalid streams=%v, %v, %v", len(m.Streams), m.String(), str) + } + + // Note that HLS score is low, so we only check duration. Note that only check half of duration, because we + // might get only some pieces of segments. + if dv := m.Duration(); dv < duration/2 { + r4 = errors.Errorf("short duration=%v < %v, %v, %v", dv, duration/2, m.String(), str) + } + } +} diff --git a/trunk/3rdparty/srs-bench/blackbox/util.go b/trunk/3rdparty/srs-bench/blackbox/util.go index ff9d91424..53a6e9d29 100644 --- a/trunk/3rdparty/srs-bench/blackbox/util.go +++ b/trunk/3rdparty/srs-bench/blackbox/util.go @@ -568,6 +568,8 @@ func (v *srsServer) Run(ctx context.Context, cancel context.CancelFunc) error { "SRS_VHOST_HLS_HLS_PATH=./objs/nginx/html", "SRS_VHOST_HLS_HLS_M3U8_FILE=[app]/[stream].m3u8", "SRS_VHOST_HLS_HLS_TS_FILE=[app]/[stream]-[seq].ts", + "SRS_VHOST_HLS_HLS_FMP4_FILE=[app]/[stream]-[seq].m4s", + "SRS_VHOST_HLS_HLS_INIT_FILE=[app]/[stream]-init.mp4", }...) // For variables. v.process.env = append(v.process.env, []string{ diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index cd7df4e7f..a38286f92 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -1821,6 +1821,13 @@ vhost hls.srs.com { # default: off enabled on; + # whether to use fmp4 as container + # The default value is off, then HLS use ts as container format, + # if on, HLS use fmp4 as container format. + # Overwrite by env SRS_VHOST_HLS_HLS_USE_FMP4 for all vhosts. + # default: off + hls_use_fmp4 on; + # the hls fragment in seconds, the duration of a piece of ts. # Overwrite by env SRS_VHOST_HLS_HLS_FRAGMENT for all vhosts. # default: 10 @@ -1886,6 +1893,44 @@ vhost hls.srs.com { # Overwrite by env SRS_VHOST_HLS_HLS_TS_FILE for all vhosts. # default: [app]/[stream]-[seq].ts hls_ts_file [app]/[stream]-[seq].ts; + # the hls fmp4 file name. + # we supports some variables to generate the filename. + # [vhost], the vhost of stream. + # [app], the app of stream. + # [stream], the stream name of stream. + # [2006], replace this const to current year. + # [01], replace this const to current month. + # [02], replace this const to current date. + # [15], replace this const to current hour. + # [04], replace this const to current minute. + # [05], replace this const to current second.p + # [999], replace this const to current millisecond. + # [timestamp],replace this const to current UNIX timestamp in ms. + # [seq], the sequence number of fmp4. + # [duration], replace this const to current ts duration. + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/dvr#custom-path + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/delivery-hls#hls-config + # Overwrite by env SRS_VHOST_HLS_HLS_FMP4_FILE for all vhosts. + # default: [app]/[stream]-[seq].m4s + hls_fmp4_file [app]/[stream]-[seq].m4s; + # the hls init mp4 file name. + # we supports some variables to generate the filename. + # [vhost], the vhost of stream. + # [app], the app of stream. + # [stream], the stream name of stream. + # [2006], replace this const to current year. + # [01], replace this const to current month. + # [02], replace this const to current date. + # [15], replace this const to current hour. + # [04], replace this const to current minute. + # [05], replace this const to current second. + # [999], replace this const to current millisecond. + # [timestamp],replace this const to current UNIX timestamp in ms. + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/dvr#custom-path + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/delivery-hls#hls-config + # Overwrite by env SRS_VHOST_HLS_HLS_INIT_FILE for all vhosts. + # default: [app]/[stream]/init.mp4 + hls_init_file [app]/[stream]/init.mp4; # the hls entry prefix, which is base url of ts url. # for example, the prefix is: # http://your-server/ diff --git a/trunk/conf/hls.mp4.conf b/trunk/conf/hls.mp4.conf new file mode 100644 index 000000000..0079ba688 --- /dev/null +++ b/trunk/conf/hls.mp4.conf @@ -0,0 +1,29 @@ +# the config for srs to delivery hls +# @see https://ossrs.net/lts/zh-cn/docs/v4/doc/sample-hls +# @see full.conf for detail config. + +listen 1935; +max_connections 1000; +daemon off; +srs_log_tank console; +http_server { + enabled on; + listen 8080; + dir ./objs/nginx/html; +} +http_api { + enabled on; + listen 1985; +} +vhost __defaultVhost__ { + hls { + enabled on; + hls_use_fmp4 on; + hls_path ./objs/nginx/html; + hls_fragment 10; + hls_window 60; + hls_m3u8_file [app]/[stream].m3u8; + hls_init_file [app]/[stream]-init.mp4; + hls_fmp4_file [app]/[stream]-[seq].m4s; + } +} diff --git a/trunk/configure b/trunk/configure index fdc909631..5c22f8916 100755 --- a/trunk/configure +++ b/trunk/configure @@ -471,7 +471,7 @@ if [[ $SRS_UTEST == YES ]]; then "srs_utest_config" "srs_utest_rtmp" "srs_utest_http" "srs_utest_avc" "srs_utest_reload" "srs_utest_mp4" "srs_utest_service" "srs_utest_app" "srs_utest_rtc" "srs_utest_config2" "srs_utest_protocol" "srs_utest_protocol2" "srs_utest_kernel2" "srs_utest_protocol3" - "srs_utest_st" "srs_utest_rtc2" "srs_utest_rtc3") + "srs_utest_st" "srs_utest_rtc2" "srs_utest_rtc3" "srs_utest_fmp4") if [[ $SRS_SRT == YES ]]; then MODULE_FILES+=("srs_utest_srt") fi diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index 96cd485bb..077c4a959 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -7,6 +7,7 @@ The changelog for SRS. ## SRS 7.0 Changelog +* v7.0, 2025-08-11, Merge [#4159](https://github.com/ossrs/srs/pull/4159): Feature: Support HLS with fmp4 segment for HEVC/LLHLS. v7.0.51 (#4159) * v7.0, 2025-08-11, Merge [#4432](https://github.com/ossrs/srs/pull/4432): AI: HTTP-FLV: Fix heap-use-after-free crash during stream unmount. v7.0.50 (#4432) * v7.0, 2025-07-28, Merge [#4245](https://github.com/ossrs/srs/pull/4245): Allow Forward to be configured with Env Var. v7.0.49 (#4245) * v7.0, 2025-07-16, Merge [#4295](https://github.com/ossrs/srs/pull/4295): RTC: audio packet jitter buffer. v7.0.48 (#4295) diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index 3590d0dba..d67ab5a56 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2693,7 +2693,7 @@ srs_error_t SrsConfig::check_normal_config() && m != "hls_storage" && m != "hls_mount" && m != "hls_td_ratio" && m != "hls_aof_ratio" && m != "hls_acodec" && m != "hls_vcodec" && m != "hls_m3u8_file" && m != "hls_ts_file" && m != "hls_ts_floor" && m != "hls_cleanup" && m != "hls_nb_notify" && m != "hls_wait_keyframe" && m != "hls_dispose" && m != "hls_keys" && m != "hls_fragments_per_key" && m != "hls_key_file" - && m != "hls_key_file_path" && m != "hls_key_url" && m != "hls_dts_directly" && m != "hls_ctx" && m != "hls_ts_ctx") { + && m != "hls_key_file_path" && m != "hls_key_url" && m != "hls_dts_directly" && m != "hls_ctx" && m != "hls_ts_ctx" && m != "hls_use_fmp4" && m != "hls_fmp4_file" && m != "hls_init_file") { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.hls.%s of %s", m.c_str(), vhost->arg0().c_str()); } @@ -7051,6 +7051,31 @@ bool SrsConfig::get_hls_enabled(SrsConfDirective* vhost) return SRS_CONF_PREFER_FALSE(conf->arg0()); } +bool SrsConfig::get_hls_use_fmp4(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.hls.hls_use_fmp4"); // SRS_VHOST_HLS_HLS_USE_FMP4 + + static bool DEFAULT = false; + + SrsConfDirective* conf = get_vhost(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls"); + + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_use_fmp4"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return SRS_CONF_PREFER_FALSE(conf->arg0()); +} + string SrsConfig::get_hls_entry_prefix(string vhost) { SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_entry_prefix"); // SRS_VHOST_HLS_HLS_ENTRY_PREFIX @@ -7127,6 +7152,44 @@ string SrsConfig::get_hls_ts_file(string vhost) return conf->arg0(); } +string SrsConfig::get_hls_fmp4_file(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_fmp4_file"); // SRS_VHOST_HLS_HLS_FMP4_FILE + + static string DEFAULT = "[app]/[stream]-[seq].m4s"; + + SrsConfDirective* conf = get_hls(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_fmp4_file"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return conf->arg0(); +} + +string SrsConfig::get_hls_init_file(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_init_file"); // SRS_VHOST_HLS_HLS_INIT_FILE + + static string DEFAULT = "[app]/[stream]/init.mp4"; + + SrsConfDirective* conf = get_hls(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_init_file"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return conf->arg0(); +} + bool SrsConfig::get_hls_ts_floor(string vhost) { SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.hls.hls_ts_floor"); // SRS_VHOST_HLS_HLS_TS_FLOOR diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index 74e7617a0..1085faeb9 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -943,6 +943,8 @@ public: // Whether HLS is enabled. virtual bool get_hls_enabled(std::string vhost); virtual bool get_hls_enabled(SrsConfDirective* vhost); + // Whether HLS use fmp4 container format + virtual bool get_hls_use_fmp4(std::string vhost); // Get the HLS m3u8 list ts segment entry prefix info. virtual std::string get_hls_entry_prefix(std::string vhost); // Get the HLS ts/m3u8 file store path. @@ -951,6 +953,10 @@ public: virtual std::string get_hls_m3u8_file(std::string vhost); // Get the HLS ts file path template. virtual std::string get_hls_ts_file(std::string vhost); + // Get the HLS fmp4 file path template. + virtual std::string get_hls_fmp4_file(std::string vhost); + // Get the HLS init mp4 file path template. + virtual std::string get_hls_init_file(std::string vhost); // Whether enable the floor(timestamp/hls_fragment) for variable timestamp. virtual bool get_hls_ts_floor(std::string vhost); // Get the hls fragment time, in srs_utime_t. @@ -995,6 +1001,7 @@ public: // Whether enable hls_ctx virtual bool get_hls_ctx_enabled(std::string vhost); // Whether enable session for ts file. + // The ts file including .ts file for MPEG-ts segment, .m4s file and init.mp4 file for fmp4 segment. virtual bool get_hls_ts_ctx_enabled(std::string vhost); // hds section private: diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 1a4f9f7f7..4c7c6759d 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -31,6 +31,7 @@ using namespace std; #include #include #include +#include #include #include #include @@ -76,6 +77,186 @@ srs_error_t SrsHlsSegment::rename() return SrsFragment::rename(); } +SrsInitMp4Segment::SrsInitMp4Segment(SrsFileWriter* fw) +{ + fw_ = fw; + const_iv_size_ = 0; +} + +SrsInitMp4Segment::~SrsInitMp4Segment() +{ + fw_->close(); +} + +srs_error_t SrsInitMp4Segment::config_cipher(unsigned char* kid, unsigned char* const_iv, uint8_t const_iv_size) +{ + if (const_iv_size != 8 && const_iv_size != 16) { + return srs_error_new(ERROR_MP4_BOX_STRING, "invalidate const_iv_size=%d", const_iv_size); + } + + memcpy(kid_, kid, 16); + memcpy(const_iv_, const_iv, const_iv_size); + const_iv_size_ = const_iv_size; + + // CBCS encryption: For example, 1 encrypt block, 9 skip blocks (10% encryption) + init_.config_encryption(1, 9, kid_, const_iv, const_iv_size); + + return srs_success; +} + +srs_error_t SrsInitMp4Segment::write(SrsFormat* format, int v_tid, int a_tid) +{ + srs_error_t err = srs_success; + + if ((err = init_encoder()) != srs_success) { + return srs_error_wrap(err, "init encoder"); + } + + if ((err = init_.write(format, v_tid, a_tid)) != srs_success) { + return srs_error_wrap(err, "write init"); + } + + return err; +} + +srs_error_t SrsInitMp4Segment::write_video_only(SrsFormat* format, int v_tid) +{ + srs_error_t err = srs_success; + + if ((err = init_encoder()) != srs_success) { + return srs_error_wrap(err, "init encoder"); + } + + if ((err = init_.write(format, true, v_tid)) != srs_success) { + return srs_error_wrap(err, "write init"); + } + + return err; +} + +srs_error_t SrsInitMp4Segment::write_audio_only(SrsFormat* format, int a_tid) +{ + srs_error_t err = srs_success; + + if ((err = init_encoder()) != srs_success) { + return srs_error_wrap(err, "init encoder"); + } + + if ((err = init_.write(format, false, a_tid)) != srs_success) { + return srs_error_wrap(err, "write init"); + } + + return err; +} + +srs_error_t SrsInitMp4Segment::init_encoder() +{ + srs_error_t err = srs_success; + + srs_assert(!fullpath().empty()); + + string path_tmp = tmppath(); + if ((err = fw_->open(path_tmp)) != srs_success) { + return srs_error_wrap(err, "Open init mp4 failed, path=%s", path_tmp.c_str()); + } + + if ((err = init_.initialize(fw_)) != srs_success) { + return srs_error_wrap(err, "init"); + } + + return err; +} + +SrsHlsM4sSegment::SrsHlsM4sSegment(SrsFileWriter* fw) +{ + fw_ = fw; +} + +SrsHlsM4sSegment::~SrsHlsM4sSegment() +{ +} + +srs_error_t SrsHlsM4sSegment::initialize(int64_t time, uint32_t v_tid, uint32_t a_tid, int sequence_number, std::string m4s_path) +{ + srs_error_t err = srs_success; + + set_path(m4s_path); + + set_number(sequence_number); + if ((err = create_dir()) != srs_success) { + return srs_error_wrap(err, "create hls m4s segment dir."); + } + + if ((err = fw_->open(tmppath())) != srs_success) { + return srs_error_wrap(err, "open hls m4s segment tmp file."); + } + + if ((err = enc_.initialize(fw_, sequence_number, time, v_tid, a_tid)) != srs_success) + { + return srs_error_wrap(err, "initialize SrsFmp4SegmentEncoder"); + } + + return err; +} + +void SrsHlsM4sSegment::config_cipher(unsigned char* key, unsigned char* iv) +{ + // TODO: set key and iv to mp4 box + enc_.config_cipher(key, iv); + memcpy(this->iv, iv, 16); +} + +srs_error_t SrsHlsM4sSegment::write(SrsSharedPtrMessage* shared_msg, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (shared_msg->is_audio()) { + uint8_t* sample = (uint8_t*)format->raw; + uint32_t nb_sample = (uint32_t)format->nb_raw; + + uint32_t dts = (uint32_t)shared_msg->timestamp; + if ((err = enc_.write_sample(SrsMp4HandlerTypeSOUN, 0x00, dts, dts, sample, nb_sample)) != srs_success) { + return srs_error_wrap(err, "m4s segment write audio sample"); + } + } else if (shared_msg->is_video()) { + SrsVideoAvcFrameType frame_type = format->video->frame_type; + uint32_t cts = (uint32_t)format->video->cts; + + uint32_t dts = (uint32_t)shared_msg->timestamp; + uint32_t pts = dts + cts; + + uint8_t* sample = (uint8_t*)format->raw; + uint32_t nb_sample = (uint32_t)format->nb_raw; + if ((err = enc_.write_sample(SrsMp4HandlerTypeVIDE, frame_type, dts, pts, sample, nb_sample)) != srs_success) { + return srs_error_wrap(err, "m4s segment write video sample"); + } + } else { + srs_trace("the sample m4s segment write is neither video nor audio sample."); + return err; + } + + append(shared_msg->timestamp); + + return err; +} + +srs_error_t SrsHlsM4sSegment::reap(uint64_t dts) +{ + srs_error_t err = srs_success; + + if ((err = enc_.flush(dts)) != srs_success) { + return srs_error_wrap(err, "m4s flush encoder."); + } + + fw_->close(); + + if ((err = rename()) != srs_success) { + return srs_error_wrap(err, "m4s segment rename."); + } + + return err; +} + SrsDvrAsyncCallOnHls::SrsDvrAsyncCallOnHls(SrsContextId c, SrsRequest* r, string p, string t, string m, string mu, int s, srs_utime_t d) { req = r->copy(); @@ -182,6 +363,695 @@ string SrsDvrAsyncCallOnHlsNotify::to_string() return "on_hls_notify: " + ts_url; } +SrsHlsFmp4Muxer::SrsHlsFmp4Muxer() +{ + req_ = NULL; + hls_fragment_ = hls_window_ = 0; + hls_aof_ratio_ = 1.0; + deviation_ts_ = 0; + hls_cleanup_ = true; + hls_wait_keyframe_ = true; + previous_floor_ts_ = 0; + accept_floor_ts_ = 0; + hls_ts_floor_ = false; + max_td_ = 0; + writer_ = NULL; + sequence_no_ = 0; + current_ = NULL; + hls_keys_ = false; + hls_fragments_per_key_ = 0; + async_ = new SrsAsyncCallWorker(); + segments_ = new SrsFragmentWindow(); + latest_acodec_ = SrsAudioCodecIdForbidden; + latest_vcodec_ = SrsVideoCodecIdForbidden; + video_track_id_ = 0; + audio_track_id_ = 0; + init_mp4_ready_ = false; + video_dts_ = 0; + + memset(key_, 0, 16); + memset(iv_, 0, 16); +} + +SrsHlsFmp4Muxer::~SrsHlsFmp4Muxer() +{ + srs_freep(segments_); + srs_freep(current_); + srs_freep(req_); + srs_freep(async_); + srs_freep(writer_); +} + +void SrsHlsFmp4Muxer::dispose() +{ + srs_error_t err = srs_success; + + segments_->dispose(); + + if (current_) { + if ((err = current_->unlink_tmpfile()) != srs_success) { + srs_warn("Unlink tmp ts failed %s", srs_error_desc(err).c_str()); + srs_freep(err); + } + srs_freep(current_); + } + + if (unlink(m3u8_.c_str()) < 0) { + srs_warn("dispose unlink path failed. file=%s", m3u8_.c_str()); + } + + srs_trace("gracefully dispose hls %s", req_ ? req_->get_stream_url().c_str() : ""); +} + +int SrsHlsFmp4Muxer::sequence_no() +{ + return sequence_no_; +} + +std::string SrsHlsFmp4Muxer::m4s_url() +{ + return current_ ? current_->uri : ""; +} + +srs_utime_t SrsHlsFmp4Muxer::duration() +{ + return current_ ? current_->duration() : 0; +} + +int SrsHlsFmp4Muxer::deviation() +{ + // no floor, no deviation. + if (!hls_ts_floor_) { + return 0; + } + + return deviation_ts_; +} + +SrsAudioCodecId SrsHlsFmp4Muxer::latest_acodec() +{ + return latest_acodec_; +} + +void SrsHlsFmp4Muxer::set_latest_acodec(SrsAudioCodecId v) +{ + latest_acodec_ = v; +} + +SrsVideoCodecId SrsHlsFmp4Muxer::latest_vcodec() +{ + return latest_vcodec_; +} + +void SrsHlsFmp4Muxer::set_latest_vcodec(SrsVideoCodecId v) +{ + latest_vcodec_ = v; +} + +srs_error_t SrsHlsFmp4Muxer::initialize(int v_tid, int a_tid) +{ + video_track_id_ = v_tid; + audio_track_id_ = a_tid; + + return srs_success; +} + +srs_error_t SrsHlsFmp4Muxer::on_publish(SrsRequest* req) +{ + srs_error_t err = srs_success; + + if ((err = async_->start()) != srs_success) { + return srs_error_wrap(err, "async start"); + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_init_mp4(SrsFormat* format, bool has_video, bool has_audio) +{ + srs_error_t err = srs_success; + + std::string vhost = req_->vhost; + std::string stream = req_->stream; + std::string app = req_->app; + + // Get init.mp4 file template from configuration + std::string init_file = _srs_config->get_hls_init_file(vhost); + init_file = srs_path_build_stream(init_file, vhost, app, stream); + + std::string hls_path = _srs_config->get_hls_path(vhost); + std::string path = hls_path + "/" + init_file; + + // Create directory for the init file + std::string init_dir = srs_path_dirname(path); + if ((err = srs_create_dir_recursively(init_dir)) != srs_success) { + return srs_error_wrap(err, "Create init mp4 dir failed, dir=%s", init_dir.c_str()); + } + + SrsUniquePtr init_mp4(new SrsInitMp4Segment(writer_)); + + init_mp4->set_path(path); + + if (hls_keys_) { + init_mp4->config_cipher(kid_, iv_, 16); + } + + if (has_video && has_audio) { + if ((err = init_mp4->write(format, video_track_id_, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "write hls init.mp4 with audio and video"); + } + } else if (has_video) { + if ((err = init_mp4->write_video_only(format, video_track_id_)) != srs_success) { + return srs_error_wrap(err, "write hls init.mp4 with video only"); + } + } else if (has_audio) { + if ((err = init_mp4->write_audio_only(format, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "write hls init.mp4 with audio only"); + } + } else { + return srs_error_new(ERROR_HLS_WRITE_FAILED, "no video and no audio sequence header"); + } + + if ((err = init_mp4->rename()) != srs_success) { + return srs_error_wrap(err, "rename hls init.mp4"); + } + + // the ts url, relative or absolute url. + // TODO: FIXME: Use url and path manager. + std::string mp4_path = init_mp4->fullpath(); + if (srs_string_starts_with(mp4_path, m3u8_dir_)) { + mp4_path = mp4_path.substr(m3u8_dir_.length()); + } + while (srs_string_starts_with(mp4_path, "/")) { + mp4_path = mp4_path.substr(1); + } + + string init_mp4_uri = hls_entry_prefix_; + if (!hls_entry_prefix_.empty() && !srs_string_ends_with(hls_entry_prefix_, "/")) { + init_mp4_uri += "/"; + + // add the http dir to uri. + string http_dir = srs_path_dirname(m3u8_url_); + if (!http_dir.empty()) { + init_mp4_uri += http_dir + "/"; + } + } + init_mp4_uri += mp4_path; + + // Convert to relative URI for m3u8 playlist. + // TODO: Need to resolve the relative URI from m3u8 and init file. + init_mp4_uri_ = srs_path_basename(init_file); + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHls(_srs_context->get_id(), req_, init_mp4->fullpath(), + init_mp4_uri, m3u8_, m3u8_url_, 0, 0))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHlsNotify(_srs_context->get_id(), req_, init_mp4_uri))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + init_mp4_ready_ = true; + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (!current_) { + if ((err = segment_open(shared_audio->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + if (current_->duration() >= hls_fragment_) { + if ((err = segment_close()) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + if ((err = segment_open(shared_audio->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + current_->write(shared_audio, format); + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) +{ + srs_error_t err = srs_success; + + video_dts_ = shared_video->timestamp; + + if (!current_) { + if ((err = segment_open(shared_video->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + bool reopen = current_->duration() >= hls_fragment_; + if (reopen) { + if ((err = segment_close()) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + if ((err = segment_open(shared_video->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + current_->write(shared_video, format); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::on_unpublish() +{ + async_->stop(); + return srs_success; +} + +srs_error_t SrsHlsFmp4Muxer::update_config(SrsRequest* r) +{ + srs_error_t err = srs_success; + + srs_freep(req_); + req_ = r->copy(); + + std::string vhost = req_->vhost; + std::string stream = req_->stream; + std::string app = req_->app; + + hls_fragment_ = _srs_config->get_hls_fragment(vhost); + double hls_td_ratio = _srs_config->get_hls_td_ratio(vhost); + hls_window_ = _srs_config->get_hls_window(vhost); + + // get the hls m3u8 ts list entry prefix config + hls_entry_prefix_ = _srs_config->get_hls_entry_prefix(vhost); + // get the hls path config + hls_path_ = _srs_config->get_hls_path(vhost); + m3u8_url_ = _srs_config->get_hls_m3u8_file(vhost); + hls_m4s_file_ = _srs_config->get_hls_fmp4_file(vhost); + hls_cleanup_ = _srs_config->get_hls_cleanup(vhost); + hls_wait_keyframe_ = _srs_config->get_hls_wait_keyframe(vhost); + // the audio overflow, for pure audio to reap segment. + hls_aof_ratio_ = _srs_config->get_hls_aof_ratio(vhost); + // whether use floor(timestamp/hls_fragment) for variable timestamp + hls_ts_floor_ = _srs_config->get_hls_ts_floor(vhost); + + hls_keys_ = _srs_config->get_hls_keys(vhost); + hls_fragments_per_key_ = _srs_config->get_hls_fragments_per_key(vhost); + hls_key_file_ = _srs_config->get_hls_key_file(vhost); + hls_key_file_path_ = _srs_config->get_hls_key_file_path(vhost); + hls_key_url_ = _srs_config->get_hls_key_url(vhost); + + previous_floor_ts_ = 0; + accept_floor_ts_ = 0; + deviation_ts_ = 0; + + // generate the m3u8 dir and path. + m3u8_url_ = srs_path_build_stream(m3u8_url_, vhost, app, stream); + m3u8_ = hls_path_ + "/" + m3u8_url_; + + // when update config, reset the history target duration. + max_td_ = hls_fragment_ * hls_td_ratio; + + // create m3u8 dir once. + m3u8_dir_ = srs_path_dirname(m3u8_); + if ((err = srs_create_dir_recursively(m3u8_dir_)) != srs_success) { + return srs_error_wrap(err, "create dir"); + } + + if (hls_keys_ && (hls_path_ != hls_key_file_path_)) { + string key_file = srs_path_build_stream(hls_key_file_, vhost, app, stream); + string key_url = hls_key_file_path_ + "/" + key_file; + string key_dir = srs_path_dirname(key_url); + if ((err = srs_create_dir_recursively(key_dir)) != srs_success) { + return srs_error_wrap(err, "create dir"); + } + } + + writer_ = new SrsFileWriter(); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::segment_open(srs_utime_t basetime) +{ + srs_error_t err = srs_success; + + if (current_) { + srs_warn("ignore the segment open, for segment is already open."); + return err; + } + + // new segment. + current_ = new SrsHlsM4sSegment(writer_); + current_->sequence_no = sequence_no_++; + + if ((err = write_hls_key()) != srs_success) { + return srs_error_wrap(err, "write hls key"); + } + + // generate filename. + std::string m4s_file = hls_m4s_file_; + m4s_file = srs_path_build_stream(m4s_file, req_->vhost, req_->app, req_->stream); + if (hls_ts_floor_) { + // accept the floor ts for the first piece. + int64_t current_floor_ts = srs_update_system_time() / hls_fragment_; + if (!accept_floor_ts_) { + accept_floor_ts_ = current_floor_ts - 1; + } else { + accept_floor_ts_++; + } + + // jump when deviation more than 10p + if (accept_floor_ts_ - current_floor_ts > SRS_JUMP_WHEN_PIECE_DEVIATION) { + srs_warn("hls: jmp for ts deviation, current=%" PRId64 ", accept=%" PRId64, current_floor_ts, accept_floor_ts_); + accept_floor_ts_ = current_floor_ts - 1; + } + + // when reap ts, adjust the deviation. + deviation_ts_ = (int)(accept_floor_ts_ - current_floor_ts); + + // dup/jmp detect for ts in floor mode. + if (previous_floor_ts_ && previous_floor_ts_ != current_floor_ts - 1) { + srs_warn("hls: dup/jmp ts, previous=%" PRId64 ", current=%" PRId64 ", accept=%" PRId64 ", deviation=%d", + previous_floor_ts_, current_floor_ts, accept_floor_ts_, deviation_ts_); + } + previous_floor_ts_ = current_floor_ts; + + // we always ensure the piece is increase one by one. + std::stringstream ts_floor; + ts_floor << accept_floor_ts_; + m4s_file = srs_string_replace(m4s_file, "[timestamp]", ts_floor.str()); + + // TODO: FIMXE: we must use the accept ts floor time to generate the hour variable. + m4s_file = srs_path_build_timestamp(m4s_file); + } else { + m4s_file = srs_path_build_timestamp(m4s_file); + } + if (true) { + std::stringstream ss; + ss << current_->sequence_no; + m4s_file = srs_string_replace(m4s_file, "[seq]", ss.str()); + } + + std::string m4s_path = hls_path_ + "/" + m4s_file; + current_->set_path(m4s_path); + + // the ts url, relative or absolute url. + // TODO: FIXME: Use url and path manager. + std::string m4s_url = current_->fullpath(); + if (srs_string_starts_with(m4s_url, m3u8_dir_)) { + m4s_url = m4s_url.substr(m3u8_dir_.length()); + } + while (srs_string_starts_with(m4s_url, "/")) { + m4s_url = m4s_url.substr(1); + } + + current_->uri += hls_entry_prefix_; + if (!hls_entry_prefix_.empty() && !srs_string_ends_with(hls_entry_prefix_, "/")) { + current_->uri += "/"; + + // add the http dir to uri. + string http_dir = srs_path_dirname(m3u8_url_); + if (!http_dir.empty()) { + current_->uri += http_dir + "/"; + } + } + current_->uri += m4s_url; + + current_->initialize(basetime, video_track_id_, audio_track_id_, sequence_no_, m4s_path); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::on_sequence_header() +{ + return srs_success; +} + +bool SrsHlsFmp4Muxer::is_segment_overflow() +{ + srs_assert(current_); + + // to prevent very small segment. + if (current_->duration() < 2 * SRS_HLS_SEGMENT_MIN_DURATION) { + return false; + } + + // Use N% deviation, to smoother. + srs_utime_t deviation = hls_ts_floor_ ? SRS_HLS_FLOOR_REAP_PERCENT * deviation_ts_ * hls_fragment_ : 0; + + // Keep in mind that we use max_td for the base duration, not the hls_fragment. To calculate + // max_td, multiply hls_fragment by hls_td_ratio. + return current_->duration() >= max_td_ + deviation; +} + +bool SrsHlsFmp4Muxer::wait_keyframe() +{ + return hls_wait_keyframe_; +} + +bool SrsHlsFmp4Muxer::is_segment_absolutely_overflow() +{ + srs_assert(current_); + + // to prevent very small segment. + if (current_->duration() < 2 * SRS_HLS_SEGMENT_MIN_DURATION) { + return false; + } + + // use N% deviation, to smoother. + srs_utime_t deviation = hls_ts_floor_? SRS_HLS_FLOOR_REAP_PERCENT * deviation_ts_ * hls_fragment_ : 0; + return current_->duration() >= hls_aof_ratio_ * hls_fragment_ + deviation; +} + +void SrsHlsFmp4Muxer::update_duration(uint64_t dts) +{ + current_->append(dts / 90); +} + +srs_error_t SrsHlsFmp4Muxer::segment_close() +{ + srs_error_t err = do_segment_close(); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::do_segment_close() +{ + srs_error_t err = srs_success; + + if (!current_) { + srs_warn("ignore the segment close, for segment is not open."); + return err; + } + + if ((err = current_->reap(video_dts_)) != srs_success) { + return srs_error_wrap(err, "reap segment"); + } + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHls(_srs_context->get_id(), req_, current_->fullpath(), + current_->uri, m3u8_, m3u8_url_, current_->sequence_no, current_->duration()))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHlsNotify(_srs_context->get_id(), req_, current_->uri))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + segments_->append(current_); + current_ = NULL; + + // shrink the segments. + segments_->shrink(hls_window_); + + // refresh the m3u8, donot contains the removed ts + if ((err = refresh_m3u8()) != srs_success) { + return srs_error_wrap(err, "refresh m3u8"); + } + + // remove the ts file. + segments_->clear_expired(hls_cleanup_); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_hls_key() +{ + srs_error_t err = srs_success; + + if (hls_keys_ && current_->sequence_no % hls_fragments_per_key_ == 0) { + if (RAND_bytes(key_, 16) < 0) { + return srs_error_wrap(err, "rand key failed."); + } + if (RAND_bytes(kid_, 16) < 0) { + return srs_error_wrap(err, "rand kid failed."); + } + if (RAND_bytes(iv_, 16) < 0) { + return srs_error_wrap(err, "rand iv failed."); + } + + string key_file = srs_path_build_stream(hls_key_file_, req_->vhost, req_->app, req_->stream); + key_file = srs_string_replace(key_file, "[seq]", srs_int2str(current_->sequence_no)); + string key_url = hls_key_file_path_ + "/" + key_file; + + SrsFileWriter fw; + if ((err = fw.open(key_url)) != srs_success) { + return srs_error_wrap(err, "open file %s", key_url.c_str()); + } + + err = fw.write(key_, 16, NULL); + fw.close(); + + if (err != srs_success) { + return srs_error_wrap(err, "write key"); + } + } + + if (hls_keys_) { + current_->config_cipher(key_, iv_); + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::refresh_m3u8() +{ + srs_error_t err = srs_success; + + // no segments, also no m3u8, return. + if (segments_->empty()) { + return err; + } + + std::string temp_m3u8 = m3u8_ + ".temp"; + if ((err = _refresh_m3u8(temp_m3u8)) == srs_success) { + if (rename(temp_m3u8.c_str(), m3u8_.c_str()) < 0) { + err = srs_error_new(ERROR_HLS_WRITE_FAILED, "hls: rename m3u8 file failed. %s => %s", temp_m3u8.c_str(), m3u8_.c_str()); + } + } + + // remove the temp file. + if (srs_path_exists(temp_m3u8)) { + if (unlink(temp_m3u8.c_str()) < 0) { + srs_warn("ignore remove m3u8 failed, %s", temp_m3u8.c_str()); + } + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::_refresh_m3u8(std::string m3u8_file) +{ + srs_error_t err = srs_success; + + // no segments, return. + if (segments_->empty()) { + return err; + } + + SrsFileWriter writer; + if ((err = writer.open(m3u8_file)) != srs_success) { + return srs_error_wrap(err, "hls: open m3u8 file %s", m3u8_file.c_str()); + } + + // #EXTM3U\n + // #EXT-X-VERSION:3\n + std::stringstream ss; + ss << "#EXTM3U" << SRS_CONSTS_LF; + // TODO: for fmp4 set #EXT-X-VERSION:7, need support tag #EXT-X-MAP:URI="init.mp4", which + // at least version:5 + // DOC: https://developer.apple.com/documentation/http-live-streaming/about-the-ext-x-version-tag + ss << "#EXT-X-VERSION:7" << SRS_CONSTS_LF; + + // #EXT-X-MEDIA-SEQUENCE:4294967295\n + SrsHlsM4sSegment* first = dynamic_cast(segments_->first()); + if (first == NULL) { + return srs_error_new(ERROR_HLS_WRITE_FAILED, "segments cast"); + } + + ss << "#EXT-X-MEDIA-SEQUENCE:" << first->sequence_no << SRS_CONSTS_LF; + + // #EXT-X-TARGETDURATION:4294967295\n + /** + * @see hls-m3u8-draft-pantos-http-live-streaming-12.pdf, page 25 + * The Media Playlist file MUST contain an EXT-X-TARGETDURATION tag. + * Its value MUST be equal to or greater than the EXTINF duration of any + * media segment that appears or will appear in the Playlist file, + * rounded to the nearest integer. Its value MUST NOT change. A + * typical target duration is 10 seconds. + */ + srs_utime_t max_duration = segments_->max_duration(); + int target_duration = (int)ceil(srsu2msi(srs_max(max_duration, max_td_)) / 1000.0); + + ss << "#EXT-X-TARGETDURATION:" << target_duration << SRS_CONSTS_LF; + + // TODO: add #EXT-X-MAP:URI="init.mp4" for fmp4 + ss << "#EXT-X-MAP:URI=\"" << init_mp4_uri_ << "\"" << SRS_CONSTS_LF; + + // write all segments + for (int i = 0; i < segments_->size(); i++) { + SrsHlsM4sSegment* segment = dynamic_cast(segments_->at(i)); + + if (segment->is_sequence_header()) { + // #EXT-X-DISCONTINUITY\n + ss << "#EXT-X-DISCONTINUITY" << SRS_CONSTS_LF; + } + +#if 1 + if(hls_keys_ && ((segment->sequence_no % hls_fragments_per_key_) == 0)) { + char hexiv[33]; + srs_data_to_hex(hexiv, segment->iv, 16); + hexiv[32] = '\0'; + + string key_file = srs_path_build_stream(hls_key_file_, req_->vhost, req_->app, req_->stream); + key_file = srs_string_replace(key_file, "[seq]", srs_int2str(segment->sequence_no)); + + string key_path = key_file; + //if key_url is not set,only use the file name + if (!hls_key_url_.empty()) { + key_path = hls_key_url_ + key_file; + } + + ss << "#EXT-X-KEY:METHOD=SAMPLE-AES,URI=" << "\"" << key_path << "\",IV=0x" << hexiv << SRS_CONSTS_LF; + } +#endif + + // "#EXTINF:4294967295.208,\n" + ss.precision(3); + ss.setf(std::ios::fixed, std::ios::floatfield); + ss << "#EXTINF:" << srsu2msi(segment->duration()) / 1000.0 << ", no desc" << SRS_CONSTS_LF; + + // {file name}\n + // TODO get segment name in relative path. + std::string seg_uri = segment->fullpath(); + if (true) { + std::stringstream stemp; + stemp << srsu2msi(segment->duration()); + seg_uri = srs_string_replace(seg_uri, "[duration]", stemp.str()); + } + //ss << segment->uri << SRS_CONSTS_LF; + ss << srs_path_basename(seg_uri) << SRS_CONSTS_LF; + } + + // write m3u8 to writer. + std::string m3u8 = ss.str(); + if ((err = writer.write((char*)m3u8.c_str(), (int)m3u8.length(), NULL)) != srs_success) { + return srs_error_wrap(err, "hls: write m3u8"); + } + + return err; +} + SrsHlsMuxer::SrsHlsMuxer() { req = NULL; @@ -800,6 +1670,9 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) // #EXT-X-VERSION:3\n std::stringstream ss; ss << "#EXTM3U" << SRS_CONSTS_LF; + // TODO: for fmp4 set #EXT-X-VERSION:7, need support tag #EXT-X-MAP:URI="init.mp4", which + // at least version:5 + // DOC: https://developer.apple.com/documentation/http-live-streaming/about-the-ext-x-version-tag ss << "#EXT-X-VERSION:3" << SRS_CONSTS_LF; // #EXT-X-MEDIA-SEQUENCE:4294967295\n @@ -823,6 +1696,8 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) int target_duration = (int)ceil(srsu2msi(srs_max(max_duration, max_td)) / 1000.0); ss << "#EXT-X-TARGETDURATION:" << target_duration << SRS_CONSTS_LF; + + // TODO: add #EXT-X-MAP:URI="init.mp4" for fmp4 // write all segments for (int i = 0; i < segments->size(); i++) { @@ -875,10 +1750,22 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) return err; } +ISrsHlsController::ISrsHlsController() +{ +} + +ISrsHlsController::~ISrsHlsController() +{ +} + SrsHlsController::SrsHlsController() { tsmc = new SrsTsMessageCache(); muxer = new SrsHlsMuxer(); + + hls_dts_directly = false; + previous_audio_dts = 0; + aac_samples = 0; } SrsHlsController::~SrsHlsController() @@ -972,7 +1859,9 @@ srs_error_t SrsHlsController::on_publish(SrsRequest* req) } // This config item is used in SrsHls, we just log its value here. - bool hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); + // If enabled, directly turn FLV timestamp to TS DTS. + // @remark It'll be reloaded automatically, because the origin hub will republish while reloading. + hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); srs_trace("hls: win=%dms, frag=%dms, prefix=%s, path=%s, m3u8=%s, ts=%s, tdr=%.2f, aof=%.2f, floor=%d, clean=%d, waitk=%d, dispose=%dms, dts_directly=%d", srsu2msi(hls_window), srsu2msi(hls_fragment), entry_prefix.c_str(), path.c_str(), m3u8_file.c_str(), ts_file.c_str(), @@ -1000,7 +1889,7 @@ srs_error_t SrsHlsController::on_unpublish() return err; } -srs_error_t SrsHlsController::on_sequence_header() +srs_error_t SrsHlsController::on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) { // TODO: support discontinuity for the same stream // currently we reap and insert discontinity when encoder republish, @@ -1011,10 +1900,50 @@ srs_error_t SrsHlsController::on_sequence_header() return muxer->on_sequence_header(); } -srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) +srs_error_t SrsHlsController::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) { srs_error_t err = srs_success; + SrsAudioFrame* frame = format->audio; + + // Reset the aac samples counter when DTS jitter. + if (previous_audio_dts > shared_audio->timestamp) { + previous_audio_dts = shared_audio->timestamp; + aac_samples = 0; + } + // The diff duration in ms between two FLV audio packets. + int diff = ::abs((int)(shared_audio->timestamp - previous_audio_dts)); + previous_audio_dts = shared_audio->timestamp; + + // Guess the number of samples for each AAC frame. + // If samples is 1024, the sample-rate is 8000HZ, the diff should be 1024/8000s=128ms. + // If samples is 1024, the sample-rate is 44100HZ, the diff should be 1024/44100s=23ms. + // If samples is 2048, the sample-rate is 44100HZ, the diff should be 2048/44100s=46ms. + int nb_samples_per_frame = 0; + int guessNumberOfSamples = diff * srs_flv_srates[format->acodec->sound_rate] / 1000; + if (guessNumberOfSamples > 0) { + if (guessNumberOfSamples < 960) { + nb_samples_per_frame = 960; + } else if (guessNumberOfSamples < 1536) { + nb_samples_per_frame = 1024; + } else if (guessNumberOfSamples < 3072) { + nb_samples_per_frame = 2048; + } else { + nb_samples_per_frame = 4096; + } + } + + // Recalc the DTS by the samples of AAC. + aac_samples += nb_samples_per_frame; + int64_t dts = 90000 * aac_samples / srs_flv_srates[format->acodec->sound_rate]; + + // If directly turn FLV timestamp, overwrite the guessed DTS. + // @doc https://github.com/ossrs/srs/issues/1506#issuecomment-562063095 + if (hls_dts_directly) { + dts = shared_audio->timestamp * 90; + } + + // Refresh the codec ASAP. if (muxer->latest_acodec() != frame->acodec()->id) { srs_trace("HLS: Switch audio codec %d(%s) to %d(%s)", muxer->latest_acodec(), srs_audio_codec_id2str(muxer->latest_acodec()).c_str(), @@ -1023,7 +1952,7 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) } // write audio to cache. - if ((err = tsmc->cache_audio(frame, pts)) != srs_success) { + if ((err = tsmc->cache_audio(frame, dts)) != srs_success) { return srs_error_wrap(err, "hls: cache audio"); } @@ -1046,7 +1975,7 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) // for pure audio, aggregate some frame to one. // TODO: FIXME: Check whether it's necessary. if (muxer->pure_audio() && tsmc->audio) { - if (pts - tsmc->audio->start_pts < SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE) { + if (dts - tsmc->audio->start_pts < SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE) { return err; } } @@ -1062,9 +1991,11 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) return err; } -srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts) +srs_error_t SrsHlsController::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) { srs_error_t err = srs_success; + SrsVideoFrame* frame = format->video; + int64_t dts = shared_video->timestamp * 90; // Refresh the codec ASAP. if (muxer->latest_vcodec() != frame->vcodec()->id) { @@ -1142,6 +2073,171 @@ srs_error_t SrsHlsController::reap_segment() return err; } +SrsHlsMp4Controller::SrsHlsMp4Controller() +{ + has_video_sh_ = false; + has_audio_sh_ = false; + + video_track_id_ = 1; + audio_track_id_ = 2; + + audio_dts_ = 0; + video_dts_ = 0; + + req_ = NULL; + muxer_ = new SrsHlsFmp4Muxer(); +} + +SrsHlsMp4Controller::~SrsHlsMp4Controller() +{ + srs_freep(muxer_); +} + +srs_error_t SrsHlsMp4Controller::initialize() +{ + srs_error_t err = srs_success; + if ((err = muxer_->initialize(video_track_id_, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "initialize SrsHlsFmp4Muxer"); + } + + return err; +} + +void SrsHlsMp4Controller::dispose() +{ + muxer_->dispose(); +} + +srs_error_t SrsHlsMp4Controller::on_publish(SrsRequest* req) +{ + srs_error_t err = srs_success; + + req_ = req; + std::string vhost = req->vhost; + std::string stream = req->stream; + std::string app = req->app; + + // get the hls m3u8 ts list entry prefix config + std::string entry_prefix = _srs_config->get_hls_entry_prefix(vhost); + // get the hls path config + std::string path = _srs_config->get_hls_path(vhost); + std::string m3u8_file = _srs_config->get_hls_m3u8_file(vhost); + std::string ts_file = _srs_config->get_hls_ts_file(vhost); + + if ((err = muxer_->on_publish(req)) != srs_success) { + return srs_error_wrap(err, "muxer publish"); + } + + if ((err = muxer_->update_config(req)) != srs_success ) { + return srs_error_wrap(err, "hls: update config"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::on_unpublish() +{ + srs_error_t err = srs_success; + req_ = NULL; + + if ((err = muxer_->segment_close()) != srs_success) { + return srs_error_wrap(err, "hls: segment close"); + } + + if ((err = muxer_->on_unpublish()) != srs_success) { + return srs_error_wrap(err, "muxer unpublish"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) +{ + srs_error_t err = srs_success; + + // Ignore audio sequence header + if (format->is_aac_sequence_header() || format->is_mp3_sequence_header()) { + return err; + } + + audio_dts_ = shared_audio->timestamp; + + if ((err = muxer_->write_audio(shared_audio, format)) != srs_success) { + return srs_error_wrap(err, "write audio"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) +{ + srs_error_t err = srs_success; + SrsVideoFrame* frame = format->video; + + // Refresh the codec ASAP. + if (muxer_->latest_vcodec() != frame->vcodec()->id) { + srs_trace("HLS: Switch video codec %d(%s) to %d(%s)", muxer_->latest_acodec(), srs_video_codec_id2str(muxer_->latest_vcodec()).c_str(), + frame->vcodec()->id, srs_video_codec_id2str(frame->vcodec()->id).c_str()); + muxer_->set_latest_vcodec(frame->vcodec()->id); + } + + video_dts_ = shared_video->timestamp; + + if ((err = muxer_->write_video(shared_video, format)) != srs_success) { + return srs_error_wrap(err, "write video"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (req_ == NULL) { + return srs_error_new(ERROR_HLS_NO_STREAM, "no req yet"); + } + + if (msg->is_video()) { + has_video_sh_ = true; + } + + if (msg->is_audio()) { + if (format->acodec->aac_extra_data.size() == 0) { + srs_trace("the audio codec's aac extra data is empty"); + return err; + } + + has_audio_sh_ = true; + } + + if ((err = muxer_->write_init_mp4(format, has_video_sh_, has_audio_sh_)) != srs_success) { + return srs_error_wrap(err, "write init mp4"); + } + + return err; +} + +int SrsHlsMp4Controller::sequence_no() +{ + return muxer_->sequence_no(); +} + +std::string SrsHlsMp4Controller::ts_url() +{ + return muxer_->m4s_url(); +} + +srs_utime_t SrsHlsMp4Controller::duration() +{ + return muxer_->duration(); +} + +int SrsHlsMp4Controller::deviation() +{ + return muxer_->deviation(); +} + SrsHls::SrsHls() { req = NULL; @@ -1152,13 +2248,10 @@ SrsHls::SrsHls() unpublishing_ = false; async_reload_ = reloading_ = false; last_update_time = 0; - hls_dts_directly = false; - - previous_audio_dts = 0; - aac_samples = 0; jitter = new SrsRtmpJitter(); - controller = new SrsHlsController(); + // TODO: replace NULL by a dummy ISrsHlsController + controller = NULL; pprint = SrsPithyPrint::create_hls(); } @@ -1292,6 +2385,16 @@ srs_error_t SrsHls::initialize(SrsOriginHub* h, SrsRequest* r) hub = h; req = r; + + bool is_fmp4_enabled = _srs_config->get_hls_use_fmp4(r->vhost); + + if (!controller) { + if (is_fmp4_enabled) { + controller = new SrsHlsMp4Controller(); + } else { + controller = new SrsHlsController(); + } + } if ((err = controller->initialize()) != srs_success) { return srs_error_wrap(err, "controller initialize"); @@ -1319,10 +2422,6 @@ srs_error_t SrsHls::on_publish() if ((err = controller->on_publish(req)) != srs_success) { return srs_error_wrap(err, "hls: on publish"); } - - // If enabled, directly turn FLV timestamp to TS DTS. - // @remark It'll be reloaded automatically, because the origin hub will republish while reloading. - hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); // if enabled, open the muxer. enabled = true; @@ -1367,6 +2466,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma // Ignore if no format->acodec, it means the codec is not parsed, or unknown codec. // @issue https://github.com/ossrs/srs/issues/1506#issuecomment-562079474 + // TODO: format->acodec is always not-nil, remove this check. if (!format->acodec) { return err; } @@ -1384,8 +2484,9 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma // ignore sequence header srs_assert(format->audio); - if (acodec == SrsAudioCodecIdAAC && format->audio->aac_packet_type == SrsAudioAacFrameTraitSequenceHeader) { - return controller->on_sequence_header(); + // TODO: verify mp3 play by HLS. + if (format->is_aac_sequence_header() || format->is_mp3_sequence_header()) { + return controller->on_sequence_header(audio.get(), format); } // TODO: FIXME: config the jitter of HLS. @@ -1393,45 +2494,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma return srs_error_wrap(err, "hls: jitter"); } - // Reset the aac samples counter when DTS jitter. - if (previous_audio_dts > audio->timestamp) { - previous_audio_dts = audio->timestamp; - aac_samples = 0; - } - - // The diff duration in ms between two FLV audio packets. - int diff = ::abs((int)(audio->timestamp - previous_audio_dts)); - previous_audio_dts = audio->timestamp; - - // Guess the number of samples for each AAC frame. - // If samples is 1024, the sample-rate is 8000HZ, the diff should be 1024/8000s=128ms. - // If samples is 1024, the sample-rate is 44100HZ, the diff should be 1024/44100s=23ms. - // If samples is 2048, the sample-rate is 44100HZ, the diff should be 2048/44100s=46ms. - int nb_samples_per_frame = 0; - int guessNumberOfSamples = diff * srs_flv_srates[format->acodec->sound_rate] / 1000; - if (guessNumberOfSamples > 0) { - if (guessNumberOfSamples < 960) { - nb_samples_per_frame = 960; - } else if (guessNumberOfSamples < 1536) { - nb_samples_per_frame = 1024; - } else if (guessNumberOfSamples < 3072) { - nb_samples_per_frame = 2048; - } else { - nb_samples_per_frame = 4096; - } - } - - // Recalc the DTS by the samples of AAC. - aac_samples += nb_samples_per_frame; - int64_t dts = 90000 * aac_samples / srs_flv_srates[format->acodec->sound_rate]; - - // If directly turn FLV timestamp, overwrite the guessed DTS. - // @doc https://github.com/ossrs/srs/issues/1506#issuecomment-562063095 - if (hls_dts_directly) { - dts = audio->timestamp * 90; - } - - if ((err = controller->write_audio(format->audio, dts)) != srs_success) { + if ((err = controller->write_audio(audio.get(), format)) != srs_success) { return srs_error_wrap(err, "hls: write audio"); } @@ -1469,9 +2532,11 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma return err; } - // ignore sequence header - if (format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { - return controller->on_sequence_header(); + // ignore sequence header avc and hevc + // is avc|hevc|av1 sequence header check, but av1 packet already ignored above. so it's ok to use + // below method. + if (format->is_avc_sequence_header()) { + return controller->on_sequence_header(video.get(), format); } // TODO: FIXME: config the jitter of HLS. @@ -1479,8 +2544,7 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma return srs_error_wrap(err, "hls: jitter"); } - int64_t dts = video->timestamp * 90; - if ((err = controller->write_video(format->video, dts)) != srs_success) { + if ((err = controller->write_video(video.get(), format)) != srs_success) { return srs_error_wrap(err, "hls: write video"); } diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp index 6d815a63c..4f7f98baa 100644 --- a/trunk/src/app/srs_app_hls.hpp +++ b/trunk/src/app/srs_app_hls.hpp @@ -16,6 +16,7 @@ #include #include #include +#include class SrsFormat; class SrsSharedPtrMessage; @@ -32,11 +33,13 @@ class SrsTsAacJitter; class SrsTsMessageCache; class SrsHlsSegment; class SrsTsContext; +class SrsFmp4SegmentEncoder; // The wrapper of m3u8 segment from specification: // // 3.3.2. EXTINF // The EXTINF tag specifies the duration of a media segment. +// TODO: refactor this to support fmp4 segment. class SrsHlsSegment : public SrsFragment { public: @@ -56,11 +59,60 @@ public: SrsHlsSegment(SrsTsContext* c, SrsAudioCodecId ac, SrsVideoCodecId vc, SrsFileWriter* w); virtual ~SrsHlsSegment(); public: - void config_cipher(unsigned char* key,unsigned char* iv); + void config_cipher(unsigned char* key, unsigned char* iv); // replace the placeholder virtual srs_error_t rename(); }; +class SrsInitMp4Segment : public SrsFragment +{ +private: + SrsFileWriter* fw_; + SrsMp4M2tsInitEncoder init_; +private: + // Key ID for encryption + unsigned char kid_[16]; + // Constant IV for encryption + unsigned char const_iv_[16]; + // IV size (8 or 16 bytes) + uint8_t const_iv_size_; +public: + SrsInitMp4Segment(SrsFileWriter* fw); + virtual ~SrsInitMp4Segment(); +public: + virtual srs_error_t config_cipher(unsigned char* kid, unsigned char* const_iv, uint8_t const_iv_size); + // Write the init mp4 file, with the v_tid(video track id) and a_tid (audio track id). + virtual srs_error_t write(SrsFormat* format, int v_tid, int a_tid); + virtual srs_error_t write_video_only(SrsFormat* format, int v_tid); + virtual srs_error_t write_audio_only(SrsFormat* format, int a_tid); +private: + virtual srs_error_t init_encoder(); +}; + +// TODO: merge this code with SrsFragmentedMp4 in dash +class SrsHlsM4sSegment : public SrsFragment +{ +private: + SrsFileWriter* fw_; + SrsFmp4SegmentEncoder enc_; +public: + // m4s uri in m3u8. + std::string uri; + // sequence number in m3u8. + int sequence_no; + // IV for encryption, saved in m3u8 file. + unsigned char iv[16]; +public: + SrsHlsM4sSegment(SrsFileWriter* fw); + virtual ~SrsHlsM4sSegment(); +public: + virtual srs_error_t initialize(int64_t time, uint32_t v_tid, uint32_t a_tid, int sequence_number, std::string m4s_path); + virtual void config_cipher(unsigned char* key, unsigned char* iv); + virtual srs_error_t write(SrsSharedPtrMessage* shared_msg, SrsFormat* format); + // Finalizes segment + virtual srs_error_t reap(uint64_t dts); +}; + // The hls async call: on_hls class SrsDvrAsyncCallOnHls : public ISrsAsyncCallTask { @@ -103,6 +155,7 @@ public: // // That is, user must use HlsCache, which will control the methods of muxer, // and provides HLS mechenisms. +// TODO: Rename to SrsHlsTsMuxer, for TS file only. class SrsHlsMuxer { private: @@ -217,6 +270,157 @@ private: virtual srs_error_t _refresh_m3u8(std::string m3u8_file); }; +// Mux the HLS stream(m3u8 and m4s files). +// Generally, the m3u8 muxer only provides methods to open/close segments, +// to flush video/audio, without any mechenisms. +class SrsHlsFmp4Muxer +{ +private: + SrsRequest* req_; +private: + std::string hls_entry_prefix_; + std::string hls_path_; + std::string hls_m4s_file_; + bool hls_cleanup_; + bool hls_wait_keyframe_; + std::string m3u8_dir_; + double hls_aof_ratio_; + // TODO: FIXME: Use TBN 1000. + srs_utime_t hls_fragment_; + srs_utime_t hls_window_; + SrsAsyncCallWorker* async_; +private: + // Whether use floor algorithm for timestamp. + bool hls_ts_floor_; + // The deviation in piece to adjust the fragment to be more + // bigger or smaller. + int deviation_ts_; + // The previous reap floor timestamp, + // used to detect the dup or jmp or ts. + int64_t accept_floor_ts_; + int64_t previous_floor_ts_; + bool init_mp4_ready_; +private: + // Whether encrypted or not + // TODO: fmp4 encryption is not yet implemented. + // fmp4 support four kinds of protection scheme: 'cenc', 'cbc1', 'cens', 'cbcs'. + // @see: https://cdn.standards.iteh.ai/samples/84637/04ebded1a92a4c8ab9be6f419a3252ed/ISO-IEC-23001-7-2023.pdf + // But unfortunately the above link is just part of the spec, the full doc is not free. + // And Apple's doc said HLS support unencrypted and encrypted with 'cbcs'. + // @see: https://developer.apple.com/documentation/http-live-streaming/about-the-common-media-application-format-with-http-live-streaming-hls + // Another Apple doc said Encrypted fmp4 content MUST contain either a Sample Encryption Box('senc'), or both a Sample Auxiliary Information + // Sizes Box('saiz') and a Sample Auxiliary Information Offsets Box('saio'). + // @see: https://developer.apple.com/documentation/http-live-streaming/hls-authoring-specification-for-apple-devices + bool hls_keys_; + int hls_fragments_per_key_; + // The key file name + std::string hls_key_file_; + // The key file path + std::string hls_key_file_path_; + // The key file url + std::string hls_key_url_; + // The key and iv. + unsigned char key_[16]; + unsigned char kid_[16]; + unsigned char iv_[16]; + // The underlayer file writer. + SrsFileWriter* writer_; +private: + int sequence_no_; + srs_utime_t max_td_; + std::string m3u8_; + std::string m3u8_url_; + std::string init_mp4_uri_; // URI for init.mp4 in m3u8 playlist + int video_track_id_; + int audio_track_id_; + uint64_t video_dts_; +private: + // The available cached segments in m3u8. + SrsFragmentWindow* segments_; + // The current writing segment. + SrsHlsM4sSegment* current_; +private: + // Latest audio codec, parsed from stream. + SrsAudioCodecId latest_acodec_; + // Latest audio codec, parsed from stream. + SrsVideoCodecId latest_vcodec_; +public: + SrsHlsFmp4Muxer(); + virtual ~SrsHlsFmp4Muxer(); +public: + virtual void dispose(); +public: + virtual int sequence_no(); + virtual std::string m4s_url(); + virtual srs_utime_t duration(); + virtual int deviation(); +public: + SrsAudioCodecId latest_acodec(); + void set_latest_acodec(SrsAudioCodecId v); + SrsVideoCodecId latest_vcodec(); + void set_latest_vcodec(SrsVideoCodecId v); +public: + // Initialize the hls muxer. + virtual srs_error_t initialize(int v_tid, int a_tid); + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req); +public: + virtual srs_error_t write_init_mp4(SrsFormat* format, bool has_video, bool has_audio); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); +public: + virtual srs_error_t on_unpublish(); + // When publish, update the config for muxer. + virtual srs_error_t update_config(SrsRequest* r); + // Open a new segment(a new ts file) + virtual srs_error_t segment_open(srs_utime_t basetime); + virtual srs_error_t on_sequence_header(); + // Whether segment overflow, + // that is whether the current segment duration>=(the segment in config) + virtual bool is_segment_overflow(); + // Whether wait keyframe to reap the ts. + virtual bool wait_keyframe(); + // Whether segment absolutely overflow, for pure audio to reap segment, + // that is whether the current segment duration>=2*(the segment in config) + virtual bool is_segment_absolutely_overflow(); +public: + // When flushing video or audio, we update the duration. But, we should also update the + // duration before closing the segment. Keep in mind that it's fine to update the duration + // several times using the same dts timestamp. + void update_duration(uint64_t dts); + // Close segment(ts). + virtual srs_error_t segment_close(); +private: + virtual srs_error_t do_segment_close(); + virtual srs_error_t write_hls_key(); + virtual srs_error_t refresh_m3u8(); + virtual srs_error_t _refresh_m3u8(std::string m3u8_file); +}; + +// The base class for HLS controller +class ISrsHlsController +{ +public: + ISrsHlsController(); + virtual ~ISrsHlsController(); +public: + virtual srs_error_t initialize() = 0; + virtual void dispose() = 0; + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req) = 0; + virtual srs_error_t on_unpublish() = 0; +public: + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) = 0; + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) = 0; +public: + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) = 0; + virtual int sequence_no() = 0; + // TODO: maybe rename to segment_url? + virtual std::string ts_url() = 0; + virtual srs_utime_t duration() = 0; + virtual int deviation() = 0; +}; + // The hls stream cache, // use to cache hls stream and flush to hls muxer. // @@ -232,7 +436,8 @@ private: // when timestamp convert to flv tbn, it will loose precise, // so we must gather audio frame together, and recalc the timestamp @see SrsTsAacJitter, // we use a aac jitter to correct the audio pts. -class SrsHlsController +// TODO: Rename to SrsHlsTsController, for TS file only. +class SrsHlsController : public ISrsHlsController { private: // The HLS muxer to reap ts and m3u8. @@ -240,6 +445,14 @@ private: SrsHlsMuxer* muxer; // The TS cache SrsTsMessageCache* tsmc; + + // If the diff=dts-previous_audio_dts is about 23, + // that's the AAC samples is 1024, and we use the samples to calc the dts. + int64_t previous_audio_dts; + // The total aac samples. + uint64_t aac_samples; + // Whether directly turn FLV timestamp to TS DTS. + bool hls_dts_directly; public: SrsHlsController(); virtual ~SrsHlsController(); @@ -258,11 +471,11 @@ public: // must write a #EXT-X-DISCONTINUITY to m3u8. // @see: hls-m3u8-draft-pantos-http-live-streaming-12.txt // @see: 3.4.11. EXT-X-DISCONTINUITY - virtual srs_error_t on_sequence_header(); + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format); // write audio to cache, if need to flush, flush to muxer. - virtual srs_error_t write_audio(SrsAudioFrame* frame, int64_t pts); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); // write video to muxer. - virtual srs_error_t write_video(SrsVideoFrame* frame, int64_t dts); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); private: // Reopen the muxer for a new hls segment, // close current segment, open a new segment, @@ -271,12 +484,50 @@ private: virtual srs_error_t reap_segment(); }; -// Transmux RTMP stream to HLS(m3u8 and ts). +// HLS controller for fMP4 (.m4s) segments with init.mp4. +// Direct sample processing without caching, simpler than TS controller. +class SrsHlsMp4Controller : public ISrsHlsController +{ +private: + bool has_video_sh_; + bool has_audio_sh_; +private: + int video_track_id_; + int audio_track_id_; +private: + // Current audio dts. + uint64_t audio_dts_; + // Current video dts. + uint64_t video_dts_; +private: + SrsRequest* req_; +private: + SrsHlsFmp4Muxer* muxer_; +public: + SrsHlsMp4Controller(); + virtual ~SrsHlsMp4Controller(); +public: + virtual srs_error_t initialize(); + virtual void dispose(); + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req); + virtual srs_error_t on_unpublish(); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); +public: + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual int sequence_no(); + virtual std::string ts_url(); + virtual srs_utime_t duration(); + virtual int deviation(); +}; + +// Transmux RTMP stream to HLS(m3u8 and ts,fmp4). // TODO: FIXME: add utest for hls. class SrsHls { private: - SrsHlsController* controller; + ISrsHlsController* controller; private: SrsRequest* req; // Whether the HLS is enabled. @@ -290,14 +541,6 @@ private: bool reloading_; // To detect heartbeat and dispose it if configured. srs_utime_t last_update_time; -private: - // If the diff=dts-previous_audio_dts is about 23, - // that's the AAC samples is 1024, and we use the samples to calc the dts. - int64_t previous_audio_dts; - // The total aac samples. - uint64_t aac_samples; - // Whether directly turn FLV timestamp to TS DTS. - bool hls_dts_directly; private: SrsOriginHub* hub; SrsRtmpJitter* jitter; diff --git a/trunk/src/app/srs_app_http_static.cpp b/trunk/src/app/srs_app_http_static.cpp index bfa407471..55181857a 100644 --- a/trunk/src/app/srs_app_http_static.cpp +++ b/trunk/src/app/srs_app_http_static.cpp @@ -222,10 +222,11 @@ srs_error_t SrsHlsStream::serve_exists_session(ISrsHttpResponseWriter* w, ISrsHt } // Rebuild the m3u8 content, make .ts with hls_ctx. - size_t pos_ts = content.find(".ts"); static string QUERY_PREFIX = string(".ts?") + string(SRS_CONTEXT_IN_HLS) + string("="); - - if (pos_ts != string::npos) { + static string M4S_QUERY_PREFIX = string(".m4s?") + string(SRS_CONTEXT_IN_HLS) + string("="); + static string INIT_MP4_QUERY_PREFIX = string("init.mp4?") + string(SRS_CONTEXT_IN_HLS) + string("="); + + if (content.find(".ts") != string::npos) { string ctx = r->query_get(SRS_CONTEXT_IN_HLS); string query = QUERY_PREFIX + ctx; @@ -236,6 +237,28 @@ srs_error_t SrsHlsStream::serve_exists_session(ISrsHttpResponseWriter* w, ISrsHt } else { content = srs_string_replace(content, ".ts", query); } + } else if (content.find(".m4s") != string::npos) { + string ctx = r->query_get(SRS_CONTEXT_IN_HLS); + string query = M4S_QUERY_PREFIX + ctx; + + size_t pos_query = content.find(".m4s?"); + if (pos_query != string::npos) { + query += "&"; + content = srs_string_replace(content, ".m4s?", query); + } else { + content = srs_string_replace(content, ".m4s", query); + } + } else if (content.find("init.mp4") != string::npos) { + string ctx = r->query_get(SRS_CONTEXT_IN_HLS); + string query = INIT_MP4_QUERY_PREFIX + ctx; + + size_t pos_query = content.find("init.mp4?"); + if (pos_query != string::npos) { + query += "&"; + content = srs_string_replace(content, "init.mp4?", query); + } else { + content = srs_string_replace(content, "init.mp4", query); + } } // Response with rebuilt content. diff --git a/trunk/src/app/srs_app_http_static.hpp b/trunk/src/app/srs_app_http_static.hpp index 5d1e243a7..acf310717 100644 --- a/trunk/src/app/srs_app_http_static.hpp +++ b/trunk/src/app/srs_app_http_static.hpp @@ -74,6 +74,7 @@ protected: virtual srs_error_t serve_mp4_stream(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath, int64_t start, int64_t end); // Support HLS streaming with pseudo session id. virtual srs_error_t serve_m3u8_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); + // the ts file including: .ts .m4s init.mp4 virtual srs_error_t serve_ts_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); }; diff --git a/trunk/src/app/srs_app_server.hpp b/trunk/src/app/srs_app_server.hpp index 801990458..d56ec6ae4 100644 --- a/trunk/src/app/srs_app_server.hpp +++ b/trunk/src/app/srs_app_server.hpp @@ -130,8 +130,10 @@ private: SrsTcpListener* https_listener_; // WebRTC over TCP listener. Please note that there is always a UDP listener by RTC server. SrsTcpListener* webrtc_listener_; +#ifdef SRS_RTSP // RTSP listener, over TCP. SrsTcpListener* rtsp_listener_; +#endif // Stream Caster for push over HTTP-FLV. SrsHttpFlvListener* stream_caster_flv_listener_; // Stream Caster for push over MPEGTS-UDP diff --git a/trunk/src/core/srs_core_version7.hpp b/trunk/src/core/srs_core_version7.hpp index 1ee7bc906..8e3a9a539 100644 --- a/trunk/src/core/srs_core_version7.hpp +++ b/trunk/src/core/srs_core_version7.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 7 #define VERSION_MINOR 0 -#define VERSION_REVISION 50 +#define VERSION_REVISION 51 #endif \ No newline at end of file diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 638bacd60..f44fa4fe4 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -1378,6 +1378,7 @@ public: public: virtual bool is_aac_sequence_header(); virtual bool is_mp3_sequence_header(); + // TODO: is avc|hevc|av1 sequence header virtual bool is_avc_sequence_header(); private: // Demux the video packet in H.264 codec. diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index 9a4ad4773..79a718b1f 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -759,15 +760,8 @@ void SrsMp4MovieFragmentBox::set_mfhd(SrsMp4MovieFragmentHeaderBox* v) boxes.push_back(v); } -SrsMp4TrackFragmentBox* SrsMp4MovieFragmentBox::traf() +void SrsMp4MovieFragmentBox::add_traf(SrsMp4TrackFragmentBox* v) { - SrsMp4Box* box = get(SrsMp4BoxTypeTRAF); - return dynamic_cast(box); -} - -void SrsMp4MovieFragmentBox::set_traf(SrsMp4TrackFragmentBox* v) -{ - remove(SrsMp4BoxTypeTRAF); boxes.push_back(v); } @@ -1647,15 +1641,8 @@ SrsMp4MovieExtendsBox::~SrsMp4MovieExtendsBox() { } -SrsMp4TrackExtendsBox* SrsMp4MovieExtendsBox::trex() +void SrsMp4MovieExtendsBox::add_trex(SrsMp4TrackExtendsBox* v) { - SrsMp4Box* box = get(SrsMp4BoxTypeTREX); - return dynamic_cast(box); -} - -void SrsMp4MovieExtendsBox::set_trex(SrsMp4TrackExtendsBox* v) -{ - remove(SrsMp4BoxTypeTREX); boxes.push_back(v); } @@ -4790,6 +4777,666 @@ stringstream& SrsMp4SegmentIndexBox::dumps_detail(stringstream& ss, SrsMp4DumpCo return ss; } +SrsMp4SampleAuxiliaryInfoSizeBox::SrsMp4SampleAuxiliaryInfoSizeBox() +{ + type = SrsMp4BoxTypeSAIZ; +} + +SrsMp4SampleAuxiliaryInfoSizeBox::~SrsMp4SampleAuxiliaryInfoSizeBox() +{ +} + +int SrsMp4SampleAuxiliaryInfoSizeBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header(); + + if (flags & 0x01) { + size += 8; // add sizeof(aux_info_type) + sizeof(aux_info_type_parameter); + } + + size += 1; // sizeof(default_sample_info_size); + size += 4; // sizeof(sample_count); + + if (default_sample_info_size == 0) { + size += sample_info_sizes.size(); + } + + return size; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoSizeBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + if (flags & 0x01) { + buf->write_4bytes(aux_info_type); + buf->write_4bytes(aux_info_type_parameter); + } + + buf->write_1bytes(default_sample_info_size); + + if (default_sample_info_size == 0) { + buf->write_4bytes(sample_info_sizes.size()); + vector::iterator it; + for (it = sample_info_sizes.begin(); it != sample_info_sizes.end(); ++it) + { + buf->write_1bytes(*it); + } + } else { + buf->write_4bytes(sample_count); + } + + return err; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoSizeBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + if (flags & 0x01) { + aux_info_type = buf->read_4bytes(); + aux_info_type_parameter = buf->read_4bytes(); + } + + default_sample_info_size = buf->read_1bytes(); + sample_count = buf->read_4bytes(); + + if (default_sample_info_size == 0) { + for (int i = 0; i < sample_count; i++) { + sample_info_sizes.push_back(buf->read_1bytes()); + } + } + + return err; +} + +std::stringstream& SrsMp4SampleAuxiliaryInfoSizeBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "default_sample_info_size=" << (int)default_sample_info_size << ", sample_count=" << sample_count; + return ss; +} + +SrsMp4SampleAuxiliaryInfoOffsetBox::SrsMp4SampleAuxiliaryInfoOffsetBox() +{ + type = SrsMp4BoxTypeSAIO; +} + +SrsMp4SampleAuxiliaryInfoOffsetBox::~SrsMp4SampleAuxiliaryInfoOffsetBox() +{ +} + +int SrsMp4SampleAuxiliaryInfoOffsetBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header(); + + if (flags & 0x01) { + size += 8; // sizeof(aux_info_type) + sizeof(aux_info_type_parameter); + } + + size += 4; // sizeof(entry_count); + if (version == 0) { + size += offsets.size() * 4; + } else { + size += offsets.size() * 8; + } + + return size; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoOffsetBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + if (flags & 0x01) { + buf->write_4bytes(aux_info_type); + buf->write_4bytes(aux_info_type_parameter); + } + + buf->write_4bytes(offsets.size()); + vector::iterator it; + for (it = offsets.begin(); it != offsets.end(); ++it) + { + if (version == 0) { + buf->write_4bytes(*it); + } else { + buf->write_8bytes(*it); + } + } + + return err; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoOffsetBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + if (flags & 0x01) { + aux_info_type = buf->read_4bytes(); + aux_info_type_parameter = buf->read_4bytes(); + } + + uint32_t entry_count = buf->read_4bytes(); + for (int i = 0; i < entry_count; i++) + { + if (version == 0) { + offsets.push_back(buf->read_4bytes()); + } else { + offsets.push_back(buf->read_8bytes()); + } + + } + + return err; +} + +std::stringstream& SrsMp4SampleAuxiliaryInfoOffsetBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "entry_count=" << offsets.size(); + return ss; +} + +SrsMp4SubSampleEncryptionInfo::SrsMp4SubSampleEncryptionInfo() +{ + bytes_of_clear_data = 0; + bytes_of_protected_data = 0; +} + +SrsMp4SubSampleEncryptionInfo::~SrsMp4SubSampleEncryptionInfo() +{ +} + +uint64_t SrsMp4SubSampleEncryptionInfo::nb_bytes() +{ + // sizeof(bytes_of_clear_data) + sizeof(bytes_of_protected_data); + return 6; +} + +srs_error_t SrsMp4SubSampleEncryptionInfo::encode(SrsBuffer* buf) +{ + buf->write_2bytes(bytes_of_clear_data); + buf->write_4bytes(bytes_of_protected_data); + + return srs_success; +} + +srs_error_t SrsMp4SubSampleEncryptionInfo::decode(SrsBuffer* buf) +{ + bytes_of_clear_data = buf->read_2bytes(); + bytes_of_protected_data = buf->read_4bytes(); + + return srs_success; +} + +std::stringstream& SrsMp4SubSampleEncryptionInfo::dumps(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "bytes_of_clear_data=" << bytes_of_clear_data << ", bytes_of_protected_data=" << bytes_of_protected_data; + return ss; +} + +SrsMp4SampleEncryptionEntry::SrsMp4SampleEncryptionEntry(SrsMp4FullBox* senc, uint8_t per_sample_iv_size) +{ + senc_ = senc; + srs_assert(per_sample_iv_size == 0 || per_sample_iv_size == 8 || per_sample_iv_size == 16); + per_sample_iv_size_ = per_sample_iv_size; + iv_ = (uint8_t*) malloc(per_sample_iv_size); +} + +SrsMp4SampleEncryptionEntry::~SrsMp4SampleEncryptionEntry() +{ + free(iv_); + iv_ = NULL; +} + +srs_error_t SrsMp4SampleEncryptionEntry::set_iv(uint8_t* iv, uint8_t iv_size) +{ + srs_assert(iv_size == per_sample_iv_size_); + memcpy(iv_, iv, iv_size); + + return srs_success; +} + +uint64_t SrsMp4SampleEncryptionEntry::nb_bytes() +{ + uint64_t size = per_sample_iv_size_; + if (senc_->flags & SrsMp4CencSampleEncryptionUseSubSample) { + size += 2; // size of subsample_count + size += subsample_infos.size() * 6; + } + + return size; +} + +srs_error_t SrsMp4SampleEncryptionEntry::encode(SrsBuffer* buf) +{ + if (per_sample_iv_size_ != 0) { + buf->write_bytes((char*) iv_, per_sample_iv_size_); + } + + if (senc_->flags & SrsMp4CencSampleEncryptionUseSubSample) { + buf->write_2bytes(subsample_infos.size()); + + vector::iterator it; + for (it = subsample_infos.begin(); it != subsample_infos.end(); ++it) { + (*it).encode(buf); + } + } + + return srs_success; +} + +srs_error_t SrsMp4SampleEncryptionEntry::decode(SrsBuffer* buf) +{ + if (per_sample_iv_size_ > 0) { + buf->read_bytes((char*)iv_, per_sample_iv_size_); + } + + if (senc_->flags & SrsMp4CencSampleEncryptionUseSubSample) { + uint16_t subsample_count = buf->read_2bytes(); + for (uint16_t i = 0; i < subsample_count; i++) { + SrsMp4SubSampleEncryptionInfo info; + info.decode(buf); + subsample_infos.push_back(info); + } + } + return srs_success; +} + +std::stringstream& SrsMp4SampleEncryptionEntry::dumps(std::stringstream& ss, SrsMp4DumpContext dc) +{ + // TODO: dump what? + ss << "iv=" << iv_ << endl; + + vector::iterator it; + for (it = subsample_infos.begin(); it != subsample_infos.end(); ++it) { + (*it).dumps(ss, dc); + ss << endl; + } + + return ss; +} + +SrsMp4SampleEncryptionBox::SrsMp4SampleEncryptionBox(uint8_t per_sample_iv_size) +{ + version = 0; + flags = SrsMp4CencSampleEncryptionUseSubSample; + type = SrsMp4BoxTypeSENC; + srs_assert(per_sample_iv_size == 0 || per_sample_iv_size == 8 || per_sample_iv_size == 16); + per_sample_iv_size_ = per_sample_iv_size; +} + +SrsMp4SampleEncryptionBox::~SrsMp4SampleEncryptionBox() +{ + vector::iterator it; + for (it = entries.begin(); it != entries.end(); it++) + { + SrsMp4SampleEncryptionEntry* entry = *it; + srs_freep(entry); + } + entries.clear(); +} + +int SrsMp4SampleEncryptionBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header() + 4; + + vector::iterator it; + for (it = entries.begin(); it < entries.end(); it++) + { + size += (*it)->nb_bytes(); + } + + return size; +} + +srs_error_t SrsMp4SampleEncryptionBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_4bytes(entries.size()); + vector::iterator it; + for (it = entries.begin(); it != entries.end(); it++) + { + (*it)->encode(buf); + } + + return err; +} + +srs_error_t SrsMp4SampleEncryptionBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + vector::iterator it; + for (it = entries.begin(); it != entries.end(); it++) + { + SrsMp4SampleEncryptionEntry* entry = *it; + srs_freep(entry); + } + entries.clear(); + + int32_t size = buf->read_4bytes(); + for (int i = 0; i < size; i++) { + SrsMp4SampleEncryptionEntry *entry = new SrsMp4SampleEncryptionEntry(this, per_sample_iv_size_); + entry->decode(buf); + entries.push_back(entry); + } + + return err; +} + +std::stringstream& SrsMp4SampleEncryptionBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "sample_count=" << entries.size() << endl; + return ss; +} + +SrsMp4ProtectionSchemeInfoBox::SrsMp4ProtectionSchemeInfoBox() +{ + type = SrsMp4BoxTypeSINF; +} + +SrsMp4ProtectionSchemeInfoBox::~SrsMp4ProtectionSchemeInfoBox() +{ +} + +SrsMp4OriginalFormatBox* SrsMp4ProtectionSchemeInfoBox::frma() +{ + SrsMp4Box* box = get(SrsMp4BoxTypeFRMA); + return dynamic_cast(box); +} + +void SrsMp4ProtectionSchemeInfoBox::set_frma(SrsMp4OriginalFormatBox* v) +{ + remove(SrsMp4BoxTypeFRMA); + boxes.push_back(v); +} + +SrsMp4SchemeTypeBox* SrsMp4ProtectionSchemeInfoBox::schm() +{ + SrsMp4Box* box = get(SrsMp4BoxTypeSCHM); + return dynamic_cast(box); +} + +void SrsMp4ProtectionSchemeInfoBox::set_schm(SrsMp4SchemeTypeBox* v) +{ + remove(SrsMp4BoxTypeSCHM); + boxes.push_back(v); +} + +SrsMp4SchemeInfoBox* SrsMp4ProtectionSchemeInfoBox::schi() +{ + SrsMp4Box* box = get(SrsMp4BoxTypeSCHI); + return dynamic_cast(box); +} + +void SrsMp4ProtectionSchemeInfoBox::set_schi(SrsMp4SchemeInfoBox* v) +{ + remove(SrsMp4BoxTypeSCHI); + boxes.push_back(v); +} + + +SrsMp4OriginalFormatBox::SrsMp4OriginalFormatBox(uint32_t original_format) +{ + type = SrsMp4BoxTypeFRMA; + data_format_ = original_format; +} + +SrsMp4OriginalFormatBox::~SrsMp4OriginalFormatBox() +{ +} + +int SrsMp4OriginalFormatBox::nb_header() +{ + return SrsMp4Box::nb_header() + 4; +} + +srs_error_t SrsMp4OriginalFormatBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_4bytes(data_format_); + + return err; +} + +srs_error_t SrsMp4OriginalFormatBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + data_format_ = buf->read_4bytes(); + + return err; +} + +std::stringstream& SrsMp4OriginalFormatBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "original format=" << data_format_ << endl; + return ss; +} + +SrsMp4SchemeTypeBox::SrsMp4SchemeTypeBox() +{ + type = SrsMp4BoxTypeSCHM; + scheme_uri_size = 0; +} + +SrsMp4SchemeTypeBox::~SrsMp4SchemeTypeBox() +{ +} + +void SrsMp4SchemeTypeBox::set_scheme_uri(char* uri, uint32_t uri_size) +{ + srs_assert(uri_size < SCHM_SCHEME_URI_MAX_SIZE); + memcpy(scheme_uri, uri, uri_size); + scheme_uri_size = uri_size; + scheme_uri[uri_size] = '\0'; +} + +int SrsMp4SchemeTypeBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header() + 4 + 4; // sizeof(scheme_type) + sizeof(scheme_version) + + if (flags & 0x01) { + size += scheme_uri_size; + } + + return size; +} + +srs_error_t SrsMp4SchemeTypeBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_4bytes(scheme_type); + buf->write_4bytes(scheme_version); + + if (flags & 0x01) { + buf->write_bytes(scheme_uri, scheme_uri_size); + buf->write_1bytes(0); + } + + return err; +} + +srs_error_t SrsMp4SchemeTypeBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + scheme_type = buf->read_4bytes(); + scheme_version = buf->read_4bytes(); + + if (flags & 0x01) { + memset(scheme_uri, 0, SCHM_SCHEME_URI_MAX_SIZE); + int s = 0; + while (s < SCHM_SCHEME_URI_MAX_SIZE-1) { + char c = buf->read_1bytes(); + scheme_uri[s] = c; + s++; + if (c == '\0') { + break; + } + } + scheme_uri_size = s; + } + + return err; +} + +std::stringstream& SrsMp4SchemeTypeBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "scheme_type=" << scheme_type << ", scheme_version=" << scheme_version << endl; + if (flags & 0x01) { + ss << "scheme_uri=" << scheme_uri << endl; + } + + return ss; +} + +SrsMp4SchemeInfoBox::SrsMp4SchemeInfoBox() +{ + type = SrsMp4BoxTypeSCHI; +} + +SrsMp4SchemeInfoBox::~SrsMp4SchemeInfoBox() +{ +} + +SrsMp4TrackEncryptionBox::SrsMp4TrackEncryptionBox() +{ + type = SrsMp4BoxTypeTENC; +} + +SrsMp4TrackEncryptionBox::~SrsMp4TrackEncryptionBox() +{ +} + +void SrsMp4TrackEncryptionBox::set_default_constant_IV(uint8_t* iv, uint8_t iv_size) +{ + srs_assert(iv_size == 8 || iv_size == 16); + memcpy(default_constant_IV, iv, iv_size); + default_constant_IV_size = iv_size; +} + +int SrsMp4TrackEncryptionBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header(); + size += 1; // sizeof(reserved) + size += 1; // sizeof(reserved_2) or sizeof(default_crypt_byte_block) + sizeof(default_skip_byte_block); + size += 1; // sizeof(default_isProtected); + size += 1; // sizeof(default_Per_Sample_IV_Size; + size += 16; // sizeof(default_KID); + if (default_is_protected == 1 && default_per_sample_IV_size == 0) { + size += 1 + default_constant_IV_size; // sizeof(default_constant_IV_size) + sizeof(default_constant_IV); + } + + return size; +} + +srs_error_t SrsMp4TrackEncryptionBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_1bytes(reserved); + if (version == 0) { + buf->write_1bytes(reserved_2); + } else { + buf->write_1bytes( (default_crypt_byte_block << 4) | (default_skip_byte_block & 0x0F)); + } + + buf->write_1bytes(default_is_protected); + buf->write_1bytes(default_per_sample_IV_size); + buf->write_bytes((char*)default_KID, 16); + if (default_is_protected == 1 && default_per_sample_IV_size == 0) { + buf->write_1bytes(default_constant_IV_size); + buf->write_bytes((char*)default_constant_IV, default_constant_IV_size); + } + + return err; +} + +srs_error_t SrsMp4TrackEncryptionBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + reserved = buf->read_1bytes(); + if (version == 0) { + reserved_2 = buf->read_1bytes(); + } else { + uint8_t v = buf->read_1bytes(); + default_crypt_byte_block = v >> 4; + default_skip_byte_block = v & 0x0f; + } + + default_is_protected = buf->read_1bytes(); + default_per_sample_IV_size = buf->read_1bytes(); + buf->read_bytes((char*)default_KID, 16); + + if (default_is_protected == 1 && default_per_sample_IV_size == 0) { + default_constant_IV_size = buf->read_1bytes(); + srs_assert(default_constant_IV_size == 8 || default_constant_IV_size == 16); + buf->read_bytes((char*) default_constant_IV, default_constant_IV_size); + } + + return err; +} + +std::stringstream& SrsMp4TrackEncryptionBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + if (version != 0) { + ss << "default_crypt_byte_block=" << default_crypt_byte_block << ", default_skip_byte_block=" << default_skip_byte_block << endl; + } + ss << "default_isProtected=" << default_is_protected << ", default_per_sample_IV_size=" << default_per_sample_IV_size << endl; + + return ss; +} + SrsMp4Sample::SrsMp4Sample() { type = SrsFrameTypeForbidden; @@ -4989,13 +5636,11 @@ srs_error_t SrsMp4SampleManager::write(SrsMp4MovieBox* moov) return err; } -srs_error_t SrsMp4SampleManager::write(SrsMp4MovieFragmentBox* moof, uint64_t dts) +srs_error_t SrsMp4SampleManager::write(SrsMp4TrackFragmentBox* traf, uint64_t dts) { srs_error_t err = srs_success; - SrsMp4TrackFragmentBox* traf = moof->traf(); SrsMp4TrackFragmentRunBox* trun = traf->trun(); - trun->flags = SrsMp4TrunFlagsDataOffset | SrsMp4TrunFlagsSampleDuration | SrsMp4TrunFlagsSampleSize | SrsMp4TrunFlagsSampleFlag | SrsMp4TrunFlagsSampleCtsOffset; @@ -6199,6 +6844,10 @@ SrsMp4ObjectType SrsMp4Encoder::get_audio_object_type() SrsMp4M2tsInitEncoder::SrsMp4M2tsInitEncoder() { writer = NULL; + crypt_byte_block_ = 0; + skip_byte_block_ = 0; + iv_size_ = 0; + is_protected_ = false; } SrsMp4M2tsInitEncoder::~SrsMp4M2tsInitEncoder() @@ -6211,6 +6860,18 @@ srs_error_t SrsMp4M2tsInitEncoder::initialize(ISrsWriter* w) return srs_success; } +void SrsMp4M2tsInitEncoder::config_encryption(uint8_t crypt_byte_block, uint8_t skip_byte_block, unsigned char* kid, unsigned char* iv, uint8_t iv_size) +{ + srs_assert(crypt_byte_block + skip_byte_block == 10); + srs_assert(iv_size == 8 || iv_size == 16); + crypt_byte_block_ = crypt_byte_block; + skip_byte_block_ = skip_byte_block; + memcpy(kid_, kid, 16); + memcpy(iv_, iv, iv_size); + iv_size_ = iv_size; + is_protected_ = true; +} + srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) { srs_error_t err = srs_success; @@ -6302,6 +6963,10 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) avc1->set_avcC(avcC); avcC->avc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(avc1)) != srs_success)) { + return srs_error_wrap(err, "encrypt avc1 box"); + } } else { SrsMp4VisualSampleEntry* hev1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); stsd->append(hev1); @@ -6314,6 +6979,10 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) hev1->set_hvcC(hvcC); hvcC->hevc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(hev1)) != srs_success)) { + return srs_error_wrap(err, "encrypt hev1 box"); + } } SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); @@ -6333,7 +7002,7 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) moov->set_mvex(mvex); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - mvex->set_trex(trex); + mvex->add_trex(trex); trex->track_ID = tid; trex->default_sample_description_index = 1; @@ -6404,6 +7073,10 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) SrsMp4EsdsBox* esds = new SrsMp4EsdsBox(); mp4a->set_esds(esds); + + if (is_protected_ && ((err = config_sample_description_encryption(mp4a)) != srs_success)) { + return srs_error_wrap(err, "encrypt mp4a box"); + } SrsMp4ES_Descriptor* es = esds->es; es->ES_ID = 0x02; @@ -6434,7 +7107,7 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) moov->set_mvex(mvex); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - mvex->set_trex(trex); + mvex->add_trex(trex); trex->track_ID = tid; trex->default_sample_description_index = 1; @@ -6448,6 +7121,317 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) return err; } +srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, int v_tid, int a_tid) +{ + srs_error_t err = srs_success; + + // Write ftyp box. + if (true) { + SrsUniquePtr ftyp(new SrsMp4FileTypeBox()); + + ftyp->major_brand = SrsMp4BoxBrandMP42; // SrsMp4BoxBrandISO5; + ftyp->minor_version = 512; + ftyp->set_compatible_brands(SrsMp4BoxBrandISO6, SrsMp4BoxBrandMP41); + + if ((err = srs_mp4_write_box(writer, ftyp.get())) != srs_success) { + return srs_error_wrap(err, "write ftyp"); + } + } + + // Write moov. + if (true) { + SrsUniquePtr moov(new SrsMp4MovieBox()); + + SrsMp4MovieHeaderBox* mvhd = new SrsMp4MovieHeaderBox(); + moov->set_mvhd(mvhd); + + mvhd->timescale = 1000; // Use tbn ms. + mvhd->duration_in_tbn = 0; + mvhd->next_track_ID = 4294967295; // 2^32 - 1 + + // write video track + if (format->vcodec) { + SrsMp4TrackBox* trak = new SrsMp4TrackBox(); + moov->add_trak(trak); + + SrsMp4TrackHeaderBox* tkhd = new SrsMp4TrackHeaderBox(); + trak->set_tkhd(tkhd); + + tkhd->track_ID = v_tid; + tkhd->duration = 0; + tkhd->width = (format->vcodec->width << 16); + tkhd->height = (format->vcodec->height << 16); + + SrsMp4MediaBox* mdia = new SrsMp4MediaBox(); + trak->set_mdia(mdia); + + SrsMp4MediaHeaderBox* mdhd = new SrsMp4MediaHeaderBox(); + mdia->set_mdhd(mdhd); + + mdhd->timescale = 1000; + mdhd->duration = 0; + mdhd->set_language0('u'); + mdhd->set_language1('n'); + mdhd->set_language2('d'); + + SrsMp4HandlerReferenceBox* hdlr = new SrsMp4HandlerReferenceBox(); + mdia->set_hdlr(hdlr); + + hdlr->handler_type = SrsMp4HandlerTypeVIDE; + hdlr->name = "VideoHandler"; + + SrsMp4MediaInformationBox* minf = new SrsMp4MediaInformationBox(); + mdia->set_minf(minf); + + SrsMp4VideoMeidaHeaderBox* vmhd = new SrsMp4VideoMeidaHeaderBox(); + minf->set_vmhd(vmhd); + + SrsMp4DataInformationBox* dinf = new SrsMp4DataInformationBox(); + minf->set_dinf(dinf); + + SrsMp4DataReferenceBox* dref = new SrsMp4DataReferenceBox(); + dinf->set_dref(dref); + + SrsMp4DataEntryBox* url = new SrsMp4DataEntryUrlBox(); + dref->append(url); + + SrsMp4SampleTableBox* stbl = new SrsMp4SampleTableBox(); + minf->set_stbl(stbl); + + SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); + stbl->set_stsd(stsd); + + if (format->vcodec->id == SrsVideoCodecIdAVC) { + SrsMp4VisualSampleEntry* avc1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); + stsd->append(avc1); + + avc1->width = format->vcodec->width; + avc1->height = format->vcodec->height; + avc1->data_reference_index = 1; + + SrsMp4AvccBox* avcC = new SrsMp4AvccBox(); + avc1->set_avcC(avcC); + + avcC->avc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(avc1)) != srs_success)) { + return srs_error_wrap(err, "encrypt avc1 box"); + } + } else { + SrsMp4VisualSampleEntry* hev1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); + stsd->append(hev1); + + hev1->width = format->vcodec->width; + hev1->height = format->vcodec->height; + hev1->data_reference_index = 1; + + SrsMp4HvcCBox* hvcC = new SrsMp4HvcCBox(); + hev1->set_hvcC(hvcC); + + hvcC->hevc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(hev1)) != srs_success)) { + return srs_error_wrap(err, "encrypt hev1 box"); + } + } + + SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); + stbl->set_stts(stts); + + SrsMp4Sample2ChunkBox* stsc = new SrsMp4Sample2ChunkBox(); + stbl->set_stsc(stsc); + + SrsMp4SampleSizeBox* stsz = new SrsMp4SampleSizeBox(); + stbl->set_stsz(stsz); + + // TODO: FIXME: need to check using stco or co64? + SrsMp4ChunkOffsetBox* stco = new SrsMp4ChunkOffsetBox(); + stbl->set_stco(stco); + } + + // write audio track + if (format->acodec) { + SrsMp4TrackBox* trak = new SrsMp4TrackBox(); + moov->add_trak(trak); + + SrsMp4TrackHeaderBox* tkhd = new SrsMp4TrackHeaderBox(); + tkhd->volume = 0x0100; + trak->set_tkhd(tkhd); + + tkhd->track_ID = a_tid; + tkhd->duration = 0; + + SrsMp4MediaBox* mdia = new SrsMp4MediaBox(); + trak->set_mdia(mdia); + + SrsMp4MediaHeaderBox* mdhd = new SrsMp4MediaHeaderBox(); + mdia->set_mdhd(mdhd); + + mdhd->timescale = 1000; + mdhd->duration = 0; + mdhd->set_language0('u'); + mdhd->set_language1('n'); + mdhd->set_language2('d'); + + SrsMp4HandlerReferenceBox* hdlr = new SrsMp4HandlerReferenceBox(); + mdia->set_hdlr(hdlr); + + hdlr->handler_type = SrsMp4HandlerTypeSOUN; + hdlr->name = "SoundHandler"; + + SrsMp4MediaInformationBox* minf = new SrsMp4MediaInformationBox(); + mdia->set_minf(minf); + + SrsMp4SoundMeidaHeaderBox* smhd = new SrsMp4SoundMeidaHeaderBox(); + minf->set_smhd(smhd); + + SrsMp4DataInformationBox* dinf = new SrsMp4DataInformationBox(); + minf->set_dinf(dinf); + + SrsMp4DataReferenceBox* dref = new SrsMp4DataReferenceBox(); + dinf->set_dref(dref); + + SrsMp4DataEntryBox* url = new SrsMp4DataEntryUrlBox(); + dref->append(url); + + SrsMp4SampleTableBox* stbl = new SrsMp4SampleTableBox(); + minf->set_stbl(stbl); + + SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); + stbl->set_stsd(stsd); + + SrsMp4AudioSampleEntry* mp4a = new SrsMp4AudioSampleEntry(); + mp4a->data_reference_index = 1; + mp4a->samplerate = uint32_t(srs_flv_srates[format->acodec->sound_rate]) << 16; + if (format->acodec->sound_size == SrsAudioSampleBits16bit) { + mp4a->samplesize = 16; + } else { + mp4a->samplesize = 8; + } + if (format->acodec->sound_type == SrsAudioChannelsStereo) { + mp4a->channelcount = 2; + } else { + mp4a->channelcount = 1; + } + stsd->append(mp4a); + + SrsMp4EsdsBox* esds = new SrsMp4EsdsBox(); + mp4a->set_esds(esds); + if (is_protected_ && ((err = config_sample_description_encryption(mp4a)) != srs_success)) { + return srs_error_wrap(err, "encrypt mp4a box."); + } + + SrsMp4ES_Descriptor* es = esds->es; + es->ES_ID = 0x02; + + SrsMp4DecoderConfigDescriptor& desc = es->decConfigDescr; + desc.objectTypeIndication = SrsMp4ObjectTypeAac; + desc.streamType = SrsMp4StreamTypeAudioStream; + srs_freep(desc.decSpecificInfo); + + SrsMp4DecoderSpecificInfo* asc = new SrsMp4DecoderSpecificInfo(); + desc.decSpecificInfo = asc; + asc->asc = format->acodec->aac_extra_data; + + SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); + stbl->set_stts(stts); + + SrsMp4Sample2ChunkBox* stsc = new SrsMp4Sample2ChunkBox(); + stbl->set_stsc(stsc); + + SrsMp4SampleSizeBox* stsz = new SrsMp4SampleSizeBox(); + stbl->set_stsz(stsz); + + // TODO: FIXME: need to check using stco or co64? + SrsMp4ChunkOffsetBox* stco = new SrsMp4ChunkOffsetBox(); + stbl->set_stco(stco); + } + + if (true) { + SrsMp4MovieExtendsBox* mvex = new SrsMp4MovieExtendsBox(); + moov->set_mvex(mvex); + + // video trex + if (format->vcodec) { + SrsMp4TrackExtendsBox* v_trex = new SrsMp4TrackExtendsBox(); + mvex->add_trex(v_trex); + + v_trex->track_ID = v_tid; + v_trex->default_sample_description_index = 1; + } + + // audio trex + if (format->acodec) { + SrsMp4TrackExtendsBox* a_trex = new SrsMp4TrackExtendsBox(); + mvex->add_trex(a_trex); + + a_trex->track_ID = a_tid; + a_trex->default_sample_description_index = 1; + } + } + + if ((err = srs_mp4_write_box(writer, moov.get())) != srs_success) { + return srs_error_wrap(err, "write moov"); + } + } + + return err; +} + +/** + * box->type = 'encv' or 'enca' + * |encv| + * | |sinf| + * | | |frma| + * | | |schm| + * | | |schi| + * | | | |tenc| + */ +srs_error_t SrsMp4M2tsInitEncoder::config_sample_description_encryption(SrsMp4SampleEntry* box) +{ + srs_error_t err = srs_success; + + bool is_video_sample = false; + SrsMp4BoxType original_type = box->type; + + if (original_type == SrsMp4BoxTypeAVC1 || original_type == SrsMp4BoxTypeHEV1) + { + box->type = SrsMp4BoxTypeENCV; + is_video_sample = true; + } else if (original_type == SrsMp4BoxTypeMP4A) { + box->type = SrsMp4BoxTypeENCA; + } else { + return srs_error_new(ERROR_MP4_BOX_ILLEGAL_TYPE, "unknown sample type 0x%x to encrypt", original_type); + } + + SrsMp4ProtectionSchemeInfoBox* sinf = new SrsMp4ProtectionSchemeInfoBox(); + box->append(sinf); + + SrsMp4OriginalFormatBox* frma = new SrsMp4OriginalFormatBox(original_type); + sinf->set_frma(frma); + + SrsMp4SchemeTypeBox* schm = new SrsMp4SchemeTypeBox(); + schm->scheme_type = SrsMp4CENSchemeCBCS; + schm->scheme_version = 0x00010000; + sinf->set_schm(schm); + + SrsMp4SchemeInfoBox* schi = new SrsMp4SchemeInfoBox(); + SrsMp4TrackEncryptionBox* tenc = new SrsMp4TrackEncryptionBox(); + tenc->version = 1; + tenc->default_crypt_byte_block = is_video_sample ? crypt_byte_block_ : 0 ; + tenc->default_skip_byte_block = is_video_sample ? skip_byte_block_ : 0; + tenc->default_is_protected = 1; + tenc->default_per_sample_IV_size = 0; + tenc->default_constant_IV_size = iv_size_; + memcpy(tenc->default_constant_IV, iv_, iv_size_); + memcpy(tenc->default_KID, kid_, 16); + + schi->append(tenc); + sinf->set_schi(schi); + + return err; +} + SrsMp4M2tsSegmentEncoder::SrsMp4M2tsSegmentEncoder() { writer = NULL; @@ -6574,7 +7558,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) mfhd->sequence_number = sequence_number; SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); - moof->set_traf(traf); + moof->add_traf(traf); SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); traf->set_tfhd(tfhd); @@ -6591,7 +7575,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); traf->set_trun(trun); - if ((err = samples->write(moof.get(), dts)) != srs_success) { + if ((err = samples->write(traf, dts)) != srs_success) { return srs_error_wrap(err, "write samples"); } @@ -6641,3 +7625,254 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) return err; } +SrsFmp4SegmentEncoder::SrsFmp4SegmentEncoder() +{ + writer_ = NULL; + sequence_number_ = 0; + decode_basetime_ = 0; + audio_track_id_ = 0; + video_track_id_ = 0; + nb_audios_ = 0; + nb_videos_ = 0; + styp_bytes_ = 0; + mdat_audio_bytes_ = 0; + mdat_video_bytes_ = 0; + audio_samples_ = new SrsMp4SampleManager(); + video_samples_ = new SrsMp4SampleManager(); + + memset(iv_,0,16); + key_ = (unsigned char*)new AES_KEY(); + do_sample_encryption_ = false; +} + +SrsFmp4SegmentEncoder::~SrsFmp4SegmentEncoder() +{ + srs_freep(audio_samples_); + srs_freep(video_samples_); + + AES_KEY* k = (AES_KEY*)key_; + srs_freep(k); +} + + +srs_error_t SrsFmp4SegmentEncoder::initialize(ISrsWriter* w, uint32_t sequence, srs_utime_t basetime, uint32_t v_tid, uint32_t a_tid) +{ + srs_error_t err = srs_success; + + writer_ = w; + sequence_number_ = sequence; + decode_basetime_ = basetime; + video_track_id_ = v_tid; + audio_track_id_ = a_tid; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::config_cipher(unsigned char* key, unsigned char* iv) +{ + srs_error_t err = srs_success; + + memcpy(this->iv_, iv, 16); + + AES_KEY* k = (AES_KEY*)this->key_; + if (AES_set_encrypt_key(key, 16 * 8, k)) { + return srs_error_new(ERROR_SYSTEM_FILE_WRITE, "set aes key failed"); + } + do_sample_encryption_ = true; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::write_sample(SrsMp4HandlerType ht, uint16_t ft, + uint32_t dts, uint32_t pts, uint8_t* sample, uint32_t nb_sample) +{ + srs_error_t err = srs_success; + + SrsMp4Sample* ps = new SrsMp4Sample(); + + if (ht == SrsMp4HandlerTypeVIDE) { + ps->type = SrsFrameTypeVideo; + ps->frame_type = (SrsVideoAvcFrameType)ft; + ps->index = nb_videos_++; + video_samples_->append(ps); + mdat_video_bytes_ += nb_sample; + } else if (ht == SrsMp4HandlerTypeSOUN) { + ps->type = SrsFrameTypeAudio; + ps->index = nb_audios_++; + audio_samples_->append(ps); + mdat_audio_bytes_ += nb_sample; + } else { + srs_freep(ps); + return err; + } + + ps->tbn = 1000; + ps->dts = dts; + ps->pts = pts; + + // We should copy the sample data, which is shared ptr from video/audio message. + // Furthermore, we do free the data when freeing the sample. + ps->data = new uint8_t[nb_sample]; + memcpy(ps->data, sample, nb_sample); + ps->nb_data = nb_sample; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::flush(uint64_t dts) +{ + srs_error_t err = srs_success; + SrsMp4TrackFragmentRunBox* video_trun = NULL; + SrsMp4TrackFragmentRunBox* audio_trun = NULL; + + if (nb_videos_ == 0 && nb_audios_ == 0) { + return srs_error_new(ERROR_MP4_ILLEGAL_MDAT, "empty samples"); + } + // Create a mdat box. + // its payload will be writen by samples, + // and we will update its header(size) when flush. + SrsUniquePtr mdat(new SrsMp4MediaDataBox()); + + SrsUniquePtr moof(new SrsMp4MovieFragmentBox()); + + SrsMp4MovieFragmentHeaderBox* mfhd = new SrsMp4MovieFragmentHeaderBox(); + moof->set_mfhd(mfhd); + mfhd->sequence_number = sequence_number_; + + // write video traf + if (mdat_video_bytes_ > 0) { + // video traf + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + moof->add_traf(traf); + + SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); + traf->set_tfhd(tfhd); + + tfhd->track_id = video_track_id_; + tfhd->flags = SrsMp4TfhdFlagsDefaultBaseIsMoof; + + SrsMp4TrackFragmentDecodeTimeBox* tfdt = new SrsMp4TrackFragmentDecodeTimeBox(); + traf->set_tfdt(tfdt); + + tfdt->version = 1; + tfdt->base_media_decode_time = srsu2ms(decode_basetime_); + + SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); + traf->set_trun(trun); + video_trun = trun; + + if ((err = video_samples_->write(traf, dts)) != srs_success) { + return srs_error_wrap(err, "write samples"); + } + + // TODO: write senc, and optional saiz & saio + if (do_sample_encryption_) { + SrsMp4SampleEncryptionBox* senc = new SrsMp4SampleEncryptionBox(0); + // video_samples_; + vector::iterator it; + // write video sample data + for (it = video_samples_->samples.begin(); it != video_samples_->samples.end(); ++it) { + // SrsMp4Sample* sample = *it; + // TODO: parse hevc|avc, nalu slice header, and calculate + // sample->data; + // sample->nb_data; + } + + traf->append(senc); + } + } + + // write audio traf + if (mdat_audio_bytes_ > 0) { + // audio traf + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + moof->add_traf(traf); + + SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); + traf->set_tfhd(tfhd); + + tfhd->track_id = audio_track_id_; + tfhd->flags = SrsMp4TfhdFlagsDefaultBaseIsMoof; + + SrsMp4TrackFragmentDecodeTimeBox* tfdt = new SrsMp4TrackFragmentDecodeTimeBox(); + traf->set_tfdt(tfdt); + + tfdt->version = 1; + tfdt->base_media_decode_time = srsu2ms(decode_basetime_); + + SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); + traf->set_trun(trun); + audio_trun = trun; + + if ((err = audio_samples_->write(traf, dts)) != srs_success) { + return srs_error_wrap(err, "write samples"); + } + + // TODO: write senc, and optional saiz & saio + if (do_sample_encryption_) { + SrsMp4SampleEncryptionBox* senc = new SrsMp4SampleEncryptionBox(0); + // this->iv_; + traf->append(senc); + } + } + + // @remark Remember the data_offset of turn is size(moof)+header(mdat) + int moof_bytes = moof->nb_bytes(); + // rewrite video data_offset + if (video_trun != NULL) { + video_trun->data_offset = (int32_t)(moof_bytes + mdat->sz_header() + 0); + } + + if (audio_trun != NULL) { + audio_trun->data_offset = (int32_t)(moof_bytes + mdat->sz_header() + mdat_video_bytes_); + } + + // srs_trace("seq: %d, moof_bytes=%d, mdat->sz_header=%d", sequence_number_, moof->nb_bytes(), mdat->sz_header()); + // srs_trace("mdat_video_bytes_ = %d, mdat_audio_bytes_ = %d", mdat_video_bytes_, mdat_audio_bytes_); + + if ((err = srs_mp4_write_box(writer_, moof.get())) != srs_success) { + return srs_error_wrap(err, "write moof"); + } + + mdat->nb_data = mdat_video_bytes_ + mdat_audio_bytes_; + // Write mdat. + if (true) { + int nb_data = mdat->sz_header(); + SrsUniquePtr data(new uint8_t[nb_data]); + + SrsUniquePtr buffer(new SrsBuffer((char*)data.get(), nb_data)); + if ((err = mdat->encode(buffer.get())) != srs_success) { + return srs_error_wrap(err, "encode mdat"); + } + + // TODO: FIXME: Ensure all bytes are writen. + if ((err = writer_->write(data.get(), nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write mdat"); + } + + vector::iterator it; + // write video sample data + for (it = video_samples_->samples.begin(); it != video_samples_->samples.end(); ++it) { + SrsMp4Sample* sample = *it; + + // TODO: FIXME: Ensure all bytes are writen. + // TODO: do cbcs encryption here. sample are nalu_length + nalu data. + if ((err = writer_->write(sample->data, sample->nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write sample"); + } + } + + // write audio sample data + for (it = audio_samples_->samples.begin(); it != audio_samples_->samples.end(); ++it) { + SrsMp4Sample* sample = *it; + + // TODO: FIXME: Ensure all bytes are writen. + // TODO: do cbcs encryption here + if ((err = writer_->write(sample->data, sample->nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write sample"); + } + } + } + + return err; +} diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index c4b1fdf53..134eaacb9 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -113,6 +113,27 @@ enum SrsMp4BoxType SrsMp4BoxTypeSIDX = 0x73696478, // 'sidx' SrsMp4BoxTypeHEV1 = 0x68657631, // 'hev1' SrsMp4BoxTypeHVCC = 0x68766343, // 'hvcC' + SrsMp4BoxTypeSENC = 0x73656e63, // 'senc' + SrsMp4BoxTypeSAIZ = 0x7361697a, // 'saiz' + SrsMp4BoxTypeSAIO = 0x7361696f, // 'saio' + SrsMp4BoxTypeENCV = 0x656e6376, // 'encv' + SrsMp4BoxTypeENCA = 0x656e6361, // 'enca' + SrsMp4BoxTypeSINF = 0x73696e66, // 'sinf' + SrsMp4BoxTypeSCHI = 0x73636869, // 'schi' + SrsMp4BoxTypeTENC = 0x74656e63, // 'tenc' + SrsMp4BoxTypeFRMA = 0x66726d61, // 'frma' + SrsMp4BoxTypeSCHM = 0x7363686d, // 'schm' +}; + +// Common encryption scheme types +// @see ISO-IEC-23001-7.pdf, 4.2 +enum SrsMp4CENSchemeType +{ + SrsMp4CENSchemeCENC = 0x63656e63, // 'cenc' + SrsMp4CENSchemeCBC1 = 0x63626331, // 'cbc1' + SrsMp4CENSchemeCENS = 0x63656e73, // 'cens' + SrsMp4CENSchemeCBCS = 0x63626373, // 'cbcs' + SrsMp4CENSchemeSVE1 = 0x73766531, // 'sve1' }; // 8.4.3.3 Semantics @@ -317,9 +338,9 @@ public: // Get the header of moof. virtual SrsMp4MovieFragmentHeaderBox* mfhd(); virtual void set_mfhd(SrsMp4MovieFragmentHeaderBox* v); - // Get the traf. - virtual SrsMp4TrackFragmentBox* traf(); - virtual void set_traf(SrsMp4TrackFragmentBox* v); + + // Let moof support more than one traf + virtual void add_traf(SrsMp4TrackFragmentBox* v); }; // 8.8.5 Movie Fragment Header Box (mfhd) @@ -499,7 +520,7 @@ class SrsMp4TrackFragmentRunBox : public SrsMp4FullBox public: // The number of samples being added in this run; also the number of rows in the following // table (the rows can be empty) - //uint32_t sample_count; + // uint32_t sample_count; // The following are optional fields public: // added to the implicit or explicit data_offset established in the track fragment header. @@ -710,8 +731,7 @@ public: virtual ~SrsMp4MovieExtendsBox(); public: // Get the track extends box. - virtual SrsMp4TrackExtendsBox* trex(); - virtual void set_trex(SrsMp4TrackExtendsBox* v); + virtual void add_trex(SrsMp4TrackExtendsBox* v); }; // 8.8.3 Track Extends Box(trex) @@ -1869,6 +1889,348 @@ public: virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); }; +// Sample auxiliary information sizes box (saiz) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.7.8, page 62 +// @see https://github.com/gpac/mp4box.js/blob/master/src/parsing/saiz.js +// Syntax +// aligned(8) class SampleAuxiliaryInformationSizesBox extends FullBox('saiz', version=0, flags) +// { +// if (flags & 1) { +// unsigned int(32) aux_info_type; +// unsigned int(32) aux_info_type_parameter; +// } +// unsigned int(8) default_sample_info_size; +// unsigned int(32) sample_count; +// if (default_sample_info_size == 0) { +// unsigned int(8) sample_info_size[sample_count]; +// } +// } +class SrsMp4SampleAuxiliaryInfoSizeBox: public SrsMp4FullBox +{ +public: + uint32_t aux_info_type; + uint32_t aux_info_type_parameter; + + uint8_t default_sample_info_size; + uint32_t sample_count; + std::vector sample_info_sizes; + +public: + SrsMp4SampleAuxiliaryInfoSizeBox(); + virtual ~SrsMp4SampleAuxiliaryInfoSizeBox(); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Sample auxiliary information offsets box (saio) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.7.9, page 63 +// @see https://github.com/gpac/mp4box.js/blob/master/src/parsing/saio.js +// Syntax +// aligned(8) class SampleAuxiliaryInformationOffsetsBox extends FullBox('saio', version, flags) +// { +// if (flags & 1) { +// unsigned int(32) aux_info_type; +// unsigned int(32) aux_info_type_parameter; +// } +// unsigned int(32) entry_count; +// if (version == 0) { +// unsigned int(32) offset[entry_count]; +// } else { +// unsigned int(64) offset[entry_count]; +// } +// } +class SrsMp4SampleAuxiliaryInfoOffsetBox: public SrsMp4FullBox +{ +public: + uint32_t aux_info_type; + uint32_t aux_info_type_parameter; + // uint32_t entry_count; + std::vector offsets; + +public: + SrsMp4SampleAuxiliaryInfoOffsetBox(); + virtual ~SrsMp4SampleAuxiliaryInfoOffsetBox(); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +enum SrsMp4CencSampleEncryptionFlags +{ + SrsMp4CencSampleEncryptionTrackDefault = 0x01, + SrsMp4CencSampleEncryptionUseSubSample = 0x02, +}; + +struct SrsMp4SubSampleEncryptionInfo : public ISrsCodec +{ + uint16_t bytes_of_clear_data; + uint32_t bytes_of_protected_data; + + SrsMp4SubSampleEncryptionInfo(); + virtual ~SrsMp4SubSampleEncryptionInfo(); + + virtual uint64_t nb_bytes(); + virtual srs_error_t encode(SrsBuffer* buf); + virtual srs_error_t decode(SrsBuffer* buf); + + virtual std::stringstream& dumps(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +class SrsMp4SampleEncryptionEntry : public ISrsCodec +{ +public: + // if flags && 0x02 + std::vector subsample_infos; + +public: + SrsMp4SampleEncryptionEntry(SrsMp4FullBox* senc, uint8_t per_sample_iv_size); + virtual ~SrsMp4SampleEncryptionEntry(); + + virtual srs_error_t set_iv(uint8_t* iv, uint8_t iv_size); + virtual uint64_t nb_bytes(); + virtual srs_error_t encode(SrsBuffer* buf); + virtual srs_error_t decode(SrsBuffer* buf); + + virtual std::stringstream& dumps(std::stringstream& ss, SrsMp4DumpContext dc); + +private: + SrsMp4FullBox* senc_; + uint8_t per_sample_iv_size_; + uint8_t* iv_; +}; + +// Sample encryption box (senc) +// @see ISO-IEC-23001-7.pdf 7.2.1 +// @see https://cdn.standards.iteh.ai/samples/84637/c960c91d60ae4da7a2f9380bd7e08642/ISO-IEC-FDIS-23001-7.pdf +// CENC SAI: sample auxiliary information associated with a sample and containing cryptographic information +// such as initialization vector or subsample information +// @see ISO-IEC-23001-7.pdf 7.2.2 +// Syntax +// aligned(8) class SampleEncryptionBox extend FullBox(`senc`, version=0, flags) +// { +// unsigned int(32) sample_count; +// { +// unsigned int(Per_Sample_IV_Size*8) InitializationVector; +// if (flags & 0x000002) +// { +// unsigned int(16) subsample_count; +// { +// unsigned int(16) BytesOfClearData; +// unsigned int(32) BytesOfProtectedData; +// } [ subsample_count ] +// } +// } [ sample_count ] +// } +class SrsMp4SampleEncryptionBox: public SrsMp4FullBox +{ +public: + std::vector entries; + +private: + uint8_t per_sample_iv_size_; + +public: + // @see ISO-IEC-23001-7.pdf 9.1 + // Per_Sample_IV_Size has supported values: 0, 8, 16. + SrsMp4SampleEncryptionBox(uint8_t per_sample_iv_size); + virtual ~SrsMp4SampleEncryptionBox(); +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Original Format Box (frma) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.2, page 81 +// aligned(8) class OriginalFormatBox(codingname) extends Box ('frma') { +// unsigned int(32) data_format = codingname; +// } +class SrsMp4OriginalFormatBox : public SrsMp4Box +{ +private: + uint32_t data_format_; + +public: + SrsMp4OriginalFormatBox(uint32_t original_format); + virtual ~SrsMp4OriginalFormatBox(); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Scheme Type Box (schm) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.5, page 81 +// aligned(8) class SchemeTypeBox extends FullBox('schm', 0, flags) { +// unsigned int(32) scheme_type; // 4CC identifying the scheme +// unsigned int(32) scheme_version; // scheme version +// if (flags & 0x000001) { +// unsigned int(8) scheme_uri[]; // browser uri +// } +// } +// @see @see ISO-IEC-23001-7.pdf 4.1 +// the scheme_version field SHALL be set to 0x00010000 (Major version 1, Minor version 0). +#define SCHM_SCHEME_URI_MAX_SIZE 128 +class SrsMp4SchemeTypeBox : public SrsMp4FullBox +{ +public: + uint32_t scheme_type; + uint32_t scheme_version; + char scheme_uri[SCHM_SCHEME_URI_MAX_SIZE]; + uint32_t scheme_uri_size; + +public: + SrsMp4SchemeTypeBox(); + virtual ~SrsMp4SchemeTypeBox(); + +public: + virtual void set_scheme_uri(char* uri, uint32_t uri_size); +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Scheme Information Box (schi) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.6, page 82 +// aligned(8) class SchemeInformationBox extends Box('schi') { +// Box scheme_specific_data[]; +// } +class SrsMp4SchemeInfoBox : public SrsMp4Box +{ +public: + SrsMp4SchemeInfoBox(); + virtual ~SrsMp4SchemeInfoBox(); +}; + +// Protection Scheme Information Box (sinf) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.1, page 80 +// aligned(8) class ProtectionSchemeInfoBox(fmt) extends Box('sinf') { +// OriginalFormatBox(fmt) original_format; // frma +// SchemeTypeBox scheme_type_box; // optional +// SchemeInformationBox info; // optional +// } +class SrsMp4ProtectionSchemeInfoBox : public SrsMp4Box +{ +public: + SrsMp4ProtectionSchemeInfoBox(); + virtual ~SrsMp4ProtectionSchemeInfoBox(); + +public: + // Get the Original Format Box (frma) + virtual SrsMp4OriginalFormatBox* frma(); + virtual void set_frma(SrsMp4OriginalFormatBox* v); + // Get the Scheme Type Box (schm) + virtual SrsMp4SchemeTypeBox* schm(); + virtual void set_schm(SrsMp4SchemeTypeBox* v); + // Get the Scheme Information Box (schi) + virtual SrsMp4SchemeInfoBox* schi(); + virtual void set_schi(SrsMp4SchemeInfoBox* v); +}; + +// Track Encryption box (tenc) +// @see ISO-IEC-23001-7.pdf 8.2 +// aligned(8) class TrackEncryptionBox extends FullBox('tenc', version, flags=0) { +// unsigned int(8) reserved = 0; +// if (version == 0) { +// unsigned int(8) reserved = 0; +// } else { // version is 1 or greater +// unsigned int(4) default_crypt_byte_block; +// unsigned int(4) default_skip_byte_block; +// } +// unsigned int(8) default_isProtected; +// unsigned int(8) default_Per_Sample_IV_Size; +// unsigned int(8)[16] default_KID; +// if (default_isProtected == 1 && default_Per_Sample_IV_Size == 0) { +// unsigned int(8) default_constant_IV_size; +// unsigned int(8)[default_constant_IV_size] default_constant_IV; +// } +// } +// @see https://developer.apple.com/documentation/http-live-streaming/about-the-common-media-application-format-with-http-live-streaming-hls +// For fragmented MPEG-4 Segments, an EXT-X-KEY tag with a METHOD=SAMPLE-AES attribute indicates that +// the Segment is encrypted using the `cbcs` scheme in ISO/IEC 23001-7. +// HLS supports unencrypted and encrypted with 'cbcs'. +// @see ISO-IEC-23001-7.pdf 10.4.1 Definition +// 'cbcs' AES-CBC subsample pattern encryption scheme. +// The 'scheme_type' field of the scheme Type Box('schm') SHALL be set to 'cbcs'. +// the version of the Track Encryption Box('tenc') SHALL be 1. +// Encrypted video tracks using NAL Structured Video conforming to ISO/IEC 14496-15 SHALL be +// protected using Subsample encryption specified in 9.5, and SHALL use pattern encryption as specified +// in 9.6. As a result, the fields crypt_byte_block and skip_byte_block SHALL NOT be 0. +// Constant IVs SHALL be used; 'default_Per_Sample_IV_Size' and 'Per_Sample_IV_Size', SHALL be 0. +// Tracks other than video are protected using whole-block full-sample encryption as specified in 9.7 and +// hence skip_byte_block SHALL be 0. +// Pattern Block length, i.e. crypt_byte_block + skip_byte_block SHOULD equal 10. +// For all video NAL units, including in 'avc1', the slice header SHALL be unencrypted. +// The first complete byte of video slice data(following the video slice header) SHALL begin a single +// Subsample protected byte range indicated by the start of BytesOfProtectedData, which extends to +// the end of the video NAL. +// NOTE 1 For AVC VCL NAL units, the encryption pattern starts at an offset rounded to the next byte after +// the slice header, i.e. on the first full byte of slice data. For HEVC, the encryption pattern starts after +// the byte_alignment() field that terminates the slice_segment_header(), i.e. on the first byte of slice data. +// +// @see ISO-IEC-23001-7.pdf 10.4.2 'cbcs' AES-CBC mode pattern encryption scheme application(informative) +// An encrypt:skip pattern of 1:9(i.e. 10% partial encryption) is recommended. Even though the syntax +// allows many different encryption patterns, a pattern of ten Blocks is recommended. This means that the +// skipped Blocks will be (10-N). The number of encrypted cipher blocks N can span multiple contiguous +// 16-byte Blocks(e.g. three encrypted Blocks followed by seven unencrypted Blocks would result in 30% +// partial encryption of the video data). +// For example, to achieve 10 % encryption, the first Block of the pattern is encrypted and the following +// nine Blocks are left unencrypted. The pattern is repeated every 160 bytes of the protected range, until +// the end of the range. If the protected range of the slice body is not a multiple of the pattern length +// (e.g. 160 bytes), then the pattern sequence applies to the included whole 16-byte Blocks and a partial +// 16-byte Block that may remain where the pattern is terminated by the byte length of the range +// BytesOfProtectedData, is left unencrypted. +// +// @see ISO-IEC-23001-7.pdf 9.7 Whole-block full sample encryption +// In whole-block full sample encryption, the entire sample is protected. Every sample is encrypted +// starting at offset 0(there is no unprotected preamble) up to the last 16-byte boundary, leaving any +// trailing 0-15 bytes in the clear. The IV is reset at every sample. +class SrsMp4TrackEncryptionBox : public SrsMp4FullBox +{ +public: + uint8_t reserved; + uint8_t reserved_2; + uint8_t default_crypt_byte_block; + uint8_t default_skip_byte_block; + uint8_t default_is_protected; + uint8_t default_per_sample_IV_size; + uint8_t default_KID[16]; + uint8_t default_constant_IV_size; + uint8_t default_constant_IV[16]; +public: + SrsMp4TrackEncryptionBox(); + virtual ~SrsMp4TrackEncryptionBox(); + +public: + virtual void set_default_constant_IV(uint8_t* iv, uint8_t iv_size); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// TODO: add SchemeTypeBox(schm), set scheme_type=cbcs + // Generally, a MP4 sample contains a frame, for example, a video frame or audio frame. class SrsMp4Sample { @@ -1931,7 +2293,7 @@ public: virtual srs_error_t write(SrsMp4MovieBox* moov); // Write the samples info to moof. // @param The dts is the dts of last segment. - virtual srs_error_t write(SrsMp4MovieFragmentBox* moof, uint64_t dts); + virtual srs_error_t write(SrsMp4TrackFragmentBox* traf, uint64_t dts); private: virtual srs_error_t write_track(SrsFrameType track, SrsMp4DecodingTime2SampleBox* stts, SrsMp4SyncSampleBox* stss, SrsMp4CompositionTime2SampleBox* ctts, @@ -2114,22 +2476,67 @@ private: }; // A fMP4 encoder, to write the init.mp4 with sequence header. +// TODO: What the M2ts short for? class SrsMp4M2tsInitEncoder { private: ISrsWriter* writer; + +private: + uint8_t crypt_byte_block_; + uint8_t skip_byte_block_; + unsigned char kid_[16]; + unsigned char iv_[16]; + uint8_t iv_size_; + bool is_protected_; + public: SrsMp4M2tsInitEncoder(); virtual ~SrsMp4M2tsInitEncoder(); public: // Initialize the encoder with a writer w. virtual srs_error_t initialize(ISrsWriter* w); + // set encryption + // TODO: review kid(map to a key) and iv, which are shared between audio/video tracks. + virtual void config_encryption(uint8_t crypt_byte_block, uint8_t skip_byte_block, unsigned char* kid, unsigned char* iv, uint8_t iv_size); // Write the sequence header. + // TODO: merge this method to its sibling. virtual srs_error_t write(SrsFormat* format, bool video, int tid); + + /** + * The mp4 box format for init.mp4. + * + * |ftyp| + * |moov| + * | |mvhd| + * | |trak| + * | |trak| + * | |....| + * | |mvex| + * | | |trex| + * | | |trex| + * | | |....| + * + * Write the sequence header with both video and audio track. + */ + virtual srs_error_t write(SrsFormat* format, int v_tid, int a_tid); + +private: + /** + * box->type = 'encv' or 'enca' + * |encv| + * | |sinf| + * | | |frma| + * | | |schm| + * | | |schi| + * | | | |tenc| + */ + virtual srs_error_t config_sample_description_encryption(SrsMp4SampleEntry* box); }; // A fMP4 encoder, to cache segments then flush to disk, because the fMP4 should write // trun box before mdat. +// TODO: fmp4 support package more than one tracks. class SrsMp4M2tsSegmentEncoder { private: @@ -2163,6 +2570,54 @@ public: virtual srs_error_t flush(uint64_t& dts); }; +// A fMP4 encoder, to cache segments then flush to disk, because the fMP4 should write +// trun box before mdat. +// TODO: fmp4 support package more than one tracks. +class SrsFmp4SegmentEncoder +{ +private: + ISrsWriter* writer_; + uint32_t sequence_number_; + // TODO: audio, video may have different basetime. + srs_utime_t decode_basetime_; + uint32_t audio_track_id_; + uint32_t video_track_id_; +private: + uint32_t nb_audios_; + uint32_t nb_videos_; + uint32_t styp_bytes_; + uint64_t mdat_audio_bytes_; + uint64_t mdat_video_bytes_; + SrsMp4SampleManager* audio_samples_; + SrsMp4SampleManager* video_samples_; +private: + // Encryption + unsigned char* key_; + unsigned char iv_[16]; + bool do_sample_encryption_; +public: + SrsFmp4SegmentEncoder(); + virtual ~SrsFmp4SegmentEncoder(); +public: + // Initialize the encoder with a writer w. + virtual srs_error_t initialize(ISrsWriter* w, uint32_t sequence, srs_utime_t basetime, uint32_t v_tid, uint32_t a_tid); + // config cipher + virtual srs_error_t config_cipher(unsigned char* key, unsigned char* iv); + // Cache a sample. + // @param ht, The sample handler type, audio/soun or video/vide. + // @param ft, The frame type. For video, it's SrsVideoAvcFrameType. + // @param dts The output dts in milliseconds. + // @param pts The output pts in milliseconds. + // @param sample The output payload, user must free it. + // @param nb_sample The output size of payload. + // @remark All samples are RAW AAC/AVC data, because sequence header is writen to init.mp4. + virtual srs_error_t write_sample(SrsMp4HandlerType ht, uint16_t ft, + uint32_t dts, uint32_t pts, uint8_t* sample, uint32_t nb_sample); + // Flush the encoder, to write the moof and mdat. + virtual srs_error_t flush(uint64_t dts); +}; + + // LCOV_EXCL_START ///////////////////////////////////////////////////////////////////////////////// // MP4 dumps functions. diff --git a/trunk/src/protocol/srs_protocol_http_stack.cpp b/trunk/src/protocol/srs_protocol_http_stack.cpp index d6c2d2907..2617bacc0 100644 --- a/trunk/src/protocol/srs_protocol_http_stack.cpp +++ b/trunk/src/protocol/srs_protocol_http_stack.cpp @@ -391,6 +391,7 @@ srs_error_t SrsHttpFileServer::serve_http(ISrsHttpResponseWriter* w, ISrsHttpMes string upath = r->path(); string fullpath = srs_http_fs_fullpath(dir, entry->pattern, upath); + string basename = srs_path_basename(upath); // stat current dir, if exists, return error. if (!_srs_path_exists(fullpath)) { @@ -400,18 +401,18 @@ srs_error_t SrsHttpFileServer::serve_http(ISrsHttpResponseWriter* w, ISrsHttpMes } srs_trace("http match file=%s, pattern=%s, upath=%s", fullpath.c_str(), entry->pattern.c_str(), upath.c_str()); - + // handle file according to its extension. // use vod stream for .flv/.fhv - if (srs_string_ends_with(fullpath, ".flv") || srs_string_ends_with(fullpath, ".fhv")) { + if (srs_string_ends_with(upath, ".flv", ".fhv")) { return serve_flv_file(w, r, fullpath); - } else if (srs_string_ends_with(fullpath, ".mp4")) { - return serve_mp4_file(w, r, fullpath); } else if (srs_string_ends_with(upath, ".m3u8")) { - return serve_m3u8_file(w, r, fullpath); - } else if (srs_string_ends_with(upath, ".ts")) { - return serve_ts_file(w, r, fullpath); - } + return serve_m3u8_ctx(w, r, fullpath); + } else if (srs_string_ends_with(upath, ".ts", ".m4s") || basename == "init.mp4") { + return serve_ts_ctx(w, r, fullpath); + } else if (srs_string_ends_with(upath, ".mp4")) { + return serve_mp4_file(w, r, fullpath); + } // serve common static file. return serve_file(w, r, fullpath); @@ -553,16 +554,6 @@ srs_error_t SrsHttpFileServer::serve_mp4_file(ISrsHttpResponseWriter* w, ISrsHtt return serve_mp4_stream(w, r, fullpath, start, end); } -srs_error_t SrsHttpFileServer::serve_m3u8_file(ISrsHttpResponseWriter * w, ISrsHttpMessage * r, std::string fullpath) -{ - return serve_m3u8_ctx(w, r, fullpath); -} - -srs_error_t SrsHttpFileServer::serve_ts_file(ISrsHttpResponseWriter * w, ISrsHttpMessage * r, std::string fullpath) -{ - return serve_ts_ctx(w, r, fullpath); -} - srs_error_t SrsHttpFileServer::serve_flv_stream(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, string fullpath, int64_t offset) { // @remark For common http file server, we don't support stream request, please use SrsVodStream instead. diff --git a/trunk/src/protocol/srs_protocol_http_stack.hpp b/trunk/src/protocol/srs_protocol_http_stack.hpp index ce25dae2e..fa6b8222e 100644 --- a/trunk/src/protocol/srs_protocol_http_stack.hpp +++ b/trunk/src/protocol/srs_protocol_http_stack.hpp @@ -351,8 +351,6 @@ private: virtual srs_error_t serve_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); virtual srs_error_t serve_flv_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); virtual srs_error_t serve_mp4_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); - virtual srs_error_t serve_m3u8_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); - virtual srs_error_t serve_ts_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); protected: // When access flv file with x.flv?start=xxx virtual srs_error_t serve_flv_stream(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath, int64_t offset); @@ -371,6 +369,7 @@ protected: // Remark 2: // If use two same "hls_ctx" in different requests, SRS cannot detect so that they will be treated as one. virtual srs_error_t serve_m3u8_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); + // the ts file including: .ts .m4s init.mp4 virtual srs_error_t serve_ts_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); protected: // Copy the fs to response writer in size bytes. diff --git a/trunk/src/utest/srs_utest_config.cpp b/trunk/src/utest/srs_utest_config.cpp index 9b06788ee..a2fe003f5 100644 --- a/trunk/src/utest/srs_utest_config.cpp +++ b/trunk/src/utest/srs_utest_config.cpp @@ -3732,12 +3732,15 @@ VOID TEST(ConfigMainTest, CheckVhostConfig5) if (true) { MockSrsConfig conf; - HELPER_ASSERT_SUCCESS(conf.parse(_MIN_OK_CONF "vhost ossrs.net{hls{hls_keys on;hls_fragments_per_key 5;hls_key_file xxx;hls_key_file_path xxx2;hls_key_url xxx3;}}")); + HELPER_ASSERT_SUCCESS(conf.parse(_MIN_OK_CONF "vhost ossrs.net{hls{hls_keys on;hls_fragments_per_key 5;hls_key_file xxx;hls_key_file_path xxx2;hls_key_url xxx3;hls_use_fmp4 on;hls_fmp4_file xx.m4s;hls_init_file yy-init.mp4;}}")); EXPECT_TRUE(conf.get_hls_keys("ossrs.net")); EXPECT_EQ(5, conf.get_hls_fragments_per_key("ossrs.net")); EXPECT_STREQ("xxx", conf.get_hls_key_file("ossrs.net").c_str()); EXPECT_STREQ("xxx2", conf.get_hls_key_file_path("ossrs.net").c_str()); EXPECT_STREQ("xxx3", conf.get_hls_key_url("ossrs.net").c_str()); + EXPECT_TRUE(conf.get_hls_use_fmp4("ossrs.net")); + EXPECT_STREQ("xx.m4s", conf.get_hls_fmp4_file("ossrs.net").c_str()); + EXPECT_STREQ("yy-init.mp4", conf.get_hls_init_file("ossrs.net").c_str()); } if (true) { @@ -5125,6 +5128,27 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesHls) SrsSetEnvConfig(conf, hls_dts_directly, "SRS_VHOST_HLS_HLS_DTS_DIRECTLY", "off"); EXPECT_FALSE(conf.get_vhost_hls_dts_directly("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_use_fmp4_on, "SRS_VHOST_HLS_HLS_USE_FMP4", "on"); + EXPECT_TRUE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_use_fmp4_off, "SRS_VHOST_HLS_HLS_USE_FMP4", "off"); + EXPECT_FALSE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_use_fmp4_unexpected, "SRS_VHOST_HLS_HLS_USE_FMP4", "xx"); + EXPECT_FALSE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_fmp4_file, "SRS_VHOST_HLS_HLS_FMP4_FILE", "xxx.m4s"); + EXPECT_STREQ("xxx.m4s", conf.get_hls_fmp4_file("__defaultVhost__").c_str()); + + SrsSetEnvConfig(conf, hls_init_file, "SRS_VHOST_HLS_HLS_INIT_FILE", "yyy-init.mp4"); + EXPECT_STREQ("yyy-init.mp4", conf.get_hls_init_file("__defaultVhost__").c_str()); + } + + // Test default value for hls_init_file with a fresh config + { + MockSrsConfig conf; + EXPECT_STREQ("[app]/[stream]/init.mp4", conf.get_hls_init_file("__defaultVhost__").c_str()); } } diff --git a/trunk/src/utest/srs_utest_fmp4.cpp b/trunk/src/utest/srs_utest_fmp4.cpp new file mode 100644 index 000000000..0aef62c17 --- /dev/null +++ b/trunk/src/utest/srs_utest_fmp4.cpp @@ -0,0 +1,810 @@ +// +// Copyright (c) 2013-2025 The SRS Authors +// +// SPDX-License-Identifier: MIT +// + +#include + +#include +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include + +// Mock classes for testing +class MockSrsRequest : public SrsRequest +{ +public: + MockSrsRequest() { + vhost = "__defaultVhost__"; + app = "live"; + stream = "livestream"; + } + virtual ~MockSrsRequest() {} +}; + +class MockSrsFormat : public SrsFormat +{ +public: + MockSrsFormat() { + initialize(); + + // Setup video sequence header (H.264 AVC) + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + on_video(0, (char*)video_raw, sizeof(video_raw)); + + // Setup audio sequence header (AAC) + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + on_audio(0, (char*)audio_raw, sizeof(audio_raw)); + } + virtual ~MockSrsFormat() {} +}; + +class MockSrsSharedPtrMessage : public SrsSharedPtrMessage +{ +public: + MockSrsSharedPtrMessage(bool is_video_msg, uint32_t ts) { + timestamp = ts; + + // Create sample payload + char* payload = new char[1024]; + memset(payload, 0x00, 1024); + SrsSharedPtrMessage::wrap(payload, 1024); + + if (is_video_msg) { + ptr->header.message_type = RTMP_MSG_VideoMessage; + } else { + ptr->header.message_type = RTMP_MSG_AudioMessage; + } + } + virtual ~MockSrsSharedPtrMessage() {} +}; + +VOID TEST(Fmp4Test, SrsInitMp4Segment_VideoOnly) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_video.mp4"); + MockSrsFormat fmt; + + HELPER_ASSERT_SUCCESS(segment.write_video_only(&fmt, 1)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify the file contains expected MP4 boxes + string content = fw.str(); + EXPECT_TRUE(content.find("ftyp") != string::npos); + EXPECT_TRUE(content.find("moov") != string::npos); +} + +VOID TEST(Fmp4Test, SrsInitMp4Segment_AudioOnly) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_audio.mp4"); + MockSrsFormat fmt; + + HELPER_ASSERT_SUCCESS(segment.write_audio_only(&fmt, 2)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify the file contains expected MP4 boxes + string content = fw.str(); + EXPECT_TRUE(content.find("ftyp") != string::npos); + EXPECT_TRUE(content.find("moov") != string::npos); +} + +VOID TEST(Fmp4Test, SrsInitMp4Segment_AudioVideo) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_av.mp4"); + MockSrsFormat fmt; + + HELPER_ASSERT_SUCCESS(segment.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify the file contains expected MP4 boxes + string content = fw.str(); + EXPECT_TRUE(content.find("ftyp") != string::npos); + EXPECT_TRUE(content.find("moov") != string::npos); +} + +VOID TEST(Fmp4Test, SrsInitMp4Segment_WithEncryption) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_encrypted.mp4"); + MockSrsFormat fmt; + + // Configure encryption + unsigned char kid[16] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}; + unsigned char iv[16] = {0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20}; + + HELPER_ASSERT_SUCCESS(segment.config_cipher(kid, iv, 16)); + HELPER_ASSERT_SUCCESS(segment.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); +} + +VOID TEST(Fmp4Test, SrsHlsM4sSegment_Basic) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsHlsM4sSegment segment(&fw); + + // Initialize segment with a path that doesn't require file system operations + HELPER_ASSERT_SUCCESS(segment.initialize(0, 1, 2, 100, "segment-100.m4s")); + + // Write video sample + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_msg(true, 1000); + HELPER_ASSERT_SUCCESS(segment.write(&video_msg, &fmt)); + + // Write audio sample + MockSrsSharedPtrMessage audio_msg(false, 2000); // Different timestamp + HELPER_ASSERT_SUCCESS(segment.write(&audio_msg, &fmt)); + + // Test duration - should be > 0 after writing samples with different timestamps + EXPECT_GT(segment.duration(), 0); +} + +VOID TEST(Fmp4Test, SrsHlsM4sSegment_WithEncryption) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsHlsM4sSegment segment(&fw); + + // Initialize segment + HELPER_ASSERT_SUCCESS(segment.initialize(0, 1, 2, 101, "segment-101.m4s")); + + // Configure encryption + unsigned char key[16] = {0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30}; + unsigned char iv[16] = {0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40}; + segment.config_cipher(key, iv); + + // Verify IV is stored + EXPECT_EQ(0x31, segment.iv[0]); + EXPECT_EQ(0x40, segment.iv[15]); + + // Write samples with different timestamps to create duration + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_msg1(true, 1000); + HELPER_ASSERT_SUCCESS(segment.write(&video_msg1, &fmt)); + + MockSrsSharedPtrMessage video_msg2(true, 2000); + HELPER_ASSERT_SUCCESS(segment.write(&video_msg2, &fmt)); + + // Test that segment has content + EXPECT_GT(segment.duration(), 0); +} + +VOID TEST(Fmp4Test, SrsFmp4SegmentEncoder_Basic) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Initialize encoder + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + // Write video sample + uint8_t video_sample[] = {0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x20}; + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + 1000, 1000, video_sample, sizeof(video_sample))); + + // Write audio sample + uint8_t audio_sample[] = {0xff, 0xf1, 0x50, 0x80, 0x01, 0x3f, 0xfc}; + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeSOUN, 0x00, + 1000, 1000, audio_sample, sizeof(audio_sample))); + + // Flush to file + HELPER_ASSERT_SUCCESS(encoder.flush(2000)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify basic structure (content may be binary, so just check size) + EXPECT_GT(fw.filesize(), 100); // Should have reasonable size for fMP4 structure +} + +VOID TEST(Fmp4Test, SrsFmp4SegmentEncoder_WithEncryption) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Initialize encoder + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + // Configure encryption + unsigned char key[16] = {0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50}; + unsigned char iv[16] = {0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60}; + encoder.config_cipher(key, iv); + + // Write encrypted samples + uint8_t video_sample[] = {0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x20}; + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + 1000, 1000, video_sample, sizeof(video_sample))); + + HELPER_ASSERT_SUCCESS(encoder.flush(2000)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify encryption boxes are present + string content = fw.str(); + EXPECT_TRUE(content.find("senc") != string::npos); // Sample Encryption Box +} + +VOID TEST(Fmp4Test, SrsHlsFmp4Muxer_Basic) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + + // Initialize muxer + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + // Test basic properties + EXPECT_EQ(0, muxer.sequence_no()); + EXPECT_EQ(0, (int)muxer.duration()); + EXPECT_EQ(0, muxer.deviation()); + + // Test codec management + muxer.set_latest_acodec(SrsAudioCodecIdAAC); + muxer.set_latest_vcodec(SrsVideoCodecIdAVC); + EXPECT_EQ(SrsAudioCodecIdAAC, muxer.latest_acodec()); + EXPECT_EQ(SrsVideoCodecIdAVC, muxer.latest_vcodec()); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsFmp4Muxer_WriteInitMp4) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(muxer.on_publish(&req)); + HELPER_ASSERT_SUCCESS(muxer.update_config(&req)); + + // Write init.mp4 with both audio and video + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, true)); + + // Write init.mp4 with video only + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, false)); + + // Write init.mp4 with audio only + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, false, true)); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsFmp4Muxer_WriteMedia) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(muxer.on_publish(&req)); + HELPER_ASSERT_SUCCESS(muxer.update_config(&req)); + + // Write init.mp4 first + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, true)); + + // Write video samples + MockSrsSharedPtrMessage video_msg(true, 1000); + HELPER_ASSERT_SUCCESS(muxer.write_video(&video_msg, &fmt)); + + // Write audio samples + MockSrsSharedPtrMessage audio_msg(false, 1000); + HELPER_ASSERT_SUCCESS(muxer.write_audio(&audio_msg, &fmt)); + + // Write more samples with time progression to accumulate duration + for (int i = 1; i <= 5; i++) { + MockSrsSharedPtrMessage video_msg2(true, 1000 + i * 1000); // 1 second increments + HELPER_ASSERT_SUCCESS(muxer.write_video(&video_msg2, &fmt)); + + MockSrsSharedPtrMessage audio_msg2(false, 1000 + i * 1000); + HELPER_ASSERT_SUCCESS(muxer.write_audio(&audio_msg2, &fmt)); + } + + // Should have accumulated duration from timestamp progression + EXPECT_GT(muxer.duration(), 0); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsMp4Controller_Basic) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + + // Initialize controller + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Test basic properties + EXPECT_EQ(0, controller.sequence_no()); + EXPECT_EQ(0, (int)controller.duration()); + + // Test URL generation (should return .m4s URL) + string url = controller.ts_url(); + EXPECT_TRUE(url.find(".m4s") != string::npos || url.empty()); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsMp4Controller_PublishWorkflow) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Publish stream + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + + // Handle sequence headers + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_sh(true, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&video_sh, &fmt)); + + MockSrsSharedPtrMessage audio_sh(false, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&audio_sh, &fmt)); + + // Write media samples + MockSrsSharedPtrMessage video_msg(true, 1000); + HELPER_ASSERT_SUCCESS(controller.write_video(&video_msg, &fmt)); + + MockSrsSharedPtrMessage audio_msg(false, 1000); + HELPER_ASSERT_SUCCESS(controller.write_audio(&audio_msg, &fmt)); + + // Unpublish + HELPER_ASSERT_SUCCESS(controller.on_unpublish()); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, SrsMp4TrackEncryptionBox_CBCS) +{ + srs_error_t err; + + SrsMp4TrackEncryptionBox tenc; + + // Configure for CBCS video encryption (1:9 pattern) + tenc.version = 1; + tenc.default_crypt_byte_block = 1; // Encrypt 1 block + tenc.default_skip_byte_block = 9; // Skip 9 blocks + tenc.default_is_protected = 1; + tenc.default_per_sample_IV_size = 0; // Use constant IV + tenc.default_constant_IV_size = 16; + + // Set Key ID + unsigned char kid[16] = {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70}; + memcpy(tenc.default_KID, kid, 16); + + // Set constant IV + unsigned char iv[16] = {0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80}; + tenc.set_default_constant_IV(iv, 16); + + // Verify configuration + EXPECT_EQ(1, tenc.version); + EXPECT_EQ(1, tenc.default_crypt_byte_block); + EXPECT_EQ(9, tenc.default_skip_byte_block); + EXPECT_EQ(1, tenc.default_is_protected); + EXPECT_EQ(0, tenc.default_per_sample_IV_size); + EXPECT_EQ(16, tenc.default_constant_IV_size); + EXPECT_EQ(0x61, tenc.default_KID[0]); + EXPECT_EQ(0x70, tenc.default_KID[15]); + EXPECT_EQ(0x71, tenc.default_constant_IV[0]); + EXPECT_EQ(0x80, tenc.default_constant_IV[15]); + + // Test encoding/decoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(tenc.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps - just verify it doesn't crash and produces some output + stringstream ss; + SrsMp4DumpContext dc; + tenc.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_FALSE(detail.empty()); // Should produce some output +} + +VOID TEST(Fmp4Test, SrsMp4TrackEncryptionBox_AudioFullSample) +{ + SrsMp4TrackEncryptionBox tenc; + + // Configure for audio full-sample encryption + tenc.version = 1; + tenc.default_crypt_byte_block = 0; // No pattern (full encryption) + tenc.default_skip_byte_block = 0; // No skip + tenc.default_is_protected = 1; + tenc.default_per_sample_IV_size = 0; // Use constant IV + tenc.default_constant_IV_size = 16; + + // Verify audio encryption configuration + EXPECT_EQ(0, tenc.default_crypt_byte_block); + EXPECT_EQ(0, tenc.default_skip_byte_block); + EXPECT_EQ(1, tenc.default_is_protected); +} + +VOID TEST(Fmp4Test, SrsMp4SampleEncryptionBox_Basic) +{ + srs_error_t err; + + SrsMp4SampleEncryptionBox senc(16); // 16-byte IV size + + // Test basic properties - flags may be set by constructor + EXPECT_EQ(0, senc.version); + EXPECT_EQ(0, (int)senc.entries.size()); + + // Test encoding empty box + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(senc.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps + stringstream ss; + SrsMp4DumpContext dc; + senc.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_TRUE(detail.find("sample_count=0") != string::npos); +} + +VOID TEST(Fmp4Test, SrsMp4SampleAuxiliaryInfoSizeBox_Basic) +{ + srs_error_t err; + + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + // Configure SAIZ box + saiz.version = 0; + saiz.flags = 0; + saiz.default_sample_info_size = 16; // 16-byte IV + saiz.sample_count = 0; + + // Test encoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(saiz.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_TRUE(detail.find("sample_count=0") != string::npos); +} + +VOID TEST(Fmp4Test, SrsMp4SampleAuxiliaryInfoOffsetBox_Basic) +{ + srs_error_t err; + + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + // Configure SAIO box + saio.version = 0; + saio.flags = 0; + saio.offsets.push_back(100); // Offset to SENC box + + // Test encoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(saio.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_TRUE(detail.find("entry_count=1") != string::npos); +} + +VOID TEST(Fmp4Test, SrsMp4OriginalFormatBox_Basic) +{ + srs_error_t err; + + SrsMp4OriginalFormatBox frma(SrsMp4BoxTypeAVC1); + + // Test encoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(frma.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps - just verify it produces output + stringstream ss; + SrsMp4DumpContext dc; + frma.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_FALSE(detail.empty()); +} + +VOID TEST(Fmp4Test, Integration_FullEncryptionWorkflow) +{ + srs_error_t err; + + // Test complete encryption workflow from init.mp4 to encrypted segments + + // 1. Create encrypted init.mp4 + MockSrsFileWriter init_fw; + SrsInitMp4Segment init_segment(&init_fw); + init_segment.set_path("encrypted_init.mp4"); + + unsigned char kid[16] = {0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, + 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90}; + unsigned char iv[16] = {0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0}; + + HELPER_ASSERT_SUCCESS(init_segment.config_cipher(kid, iv, 16)); + + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(init_segment.write(&fmt, 1, 2)); + EXPECT_TRUE(init_fw.filesize() > 0); + + // 2. Create encrypted segment + MockSrsFileWriter seg_fw; + SrsHlsM4sSegment m4s_segment(&seg_fw); + HELPER_ASSERT_SUCCESS(m4s_segment.initialize(0, 1, 2, 200, "encrypted-200.m4s")); + + unsigned char seg_key[16] = {0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0}; + unsigned char seg_iv[16] = {0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, + 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0}; + m4s_segment.config_cipher(seg_key, seg_iv); + + // Write samples to encrypted segment with time progression + MockSrsSharedPtrMessage video_msg1(true, 2000); + HELPER_ASSERT_SUCCESS(m4s_segment.write(&video_msg1, &fmt)); + + MockSrsSharedPtrMessage audio_msg1(false, 2500); + HELPER_ASSERT_SUCCESS(m4s_segment.write(&audio_msg1, &fmt)); + + MockSrsSharedPtrMessage video_msg2(true, 3000); + HELPER_ASSERT_SUCCESS(m4s_segment.write(&video_msg2, &fmt)); + + // Should have duration from timestamp progression + EXPECT_GT(m4s_segment.duration(), 0); + + // 3. Verify both files have content (encryption metadata is binary) + EXPECT_TRUE(init_fw.filesize() > 0); + EXPECT_GT(m4s_segment.duration(), 0); +} + +VOID TEST(Fmp4Test, EdgeCase_LargeTimestamp) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Test with large timestamp values + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + uint64_t large_timestamp = 0xFFFFFFFF; // Max 32-bit value + uint8_t sample[] = {0x00, 0x01, 0x02, 0x03}; + + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + large_timestamp, large_timestamp, sample, sizeof(sample))); + + HELPER_ASSERT_SUCCESS(encoder.flush(large_timestamp + 1000)); + EXPECT_TRUE(fw.filesize() > 0); +} + +VOID TEST(Fmp4Test, ErrorHandling_InvalidTrackId) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Test with track ID 0 (may or may not be invalid depending on implementation) + err = encoder.initialize(&fw, 0, 0, 0, 0); + // Just verify the function can be called without crashing + if (err != srs_success) { + srs_freep(err); + } +} + +VOID TEST(Fmp4Test, ErrorHandling_NullWriter) +{ + srs_error_t err; + + SrsFmp4SegmentEncoder encoder; + + // Initialize with null writer - may or may not fail depending on implementation + err = encoder.initialize(NULL, 0, 0, 1, 2); + if (err != srs_success) { + srs_freep(err); + } + // Test passes if no crash occurs +} + +VOID TEST(Fmp4Test, ErrorHandling_WriteBeforeInitialize) +{ + srs_error_t err; + + SrsFmp4SegmentEncoder encoder; + uint8_t sample[] = {0x00, 0x01}; + + // Try to write sample before initialization - may or may not fail + err = encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + 1000, 1000, sample, sizeof(sample)); + if (err != srs_success) { + srs_freep(err); + } + // Test passes if no crash occurs +} + +VOID TEST(Fmp4Test, ErrorHandling_FlushBeforeWrite) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + // Flush without writing any samples - may fail due to empty samples + err = encoder.flush(1000); + if (err != srs_success) { + srs_freep(err); + // This is expected behavior for empty flush + } +} + +VOID TEST(Fmp4Test, Configuration_TrackIdManagement) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Verify default track IDs + EXPECT_EQ(1, controller.video_track_id_); + EXPECT_EQ(2, controller.audio_track_id_); + + // Test sequence header tracking + EXPECT_FALSE(controller.has_video_sh_); + EXPECT_FALSE(controller.has_audio_sh_); + + // Set request first + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_sh(true, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&video_sh, &fmt)); + EXPECT_TRUE(controller.has_video_sh_); + + MockSrsSharedPtrMessage audio_sh(false, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&audio_sh, &fmt)); + EXPECT_TRUE(controller.has_audio_sh_); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, Configuration_SequenceHeaderValidation) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Test sequence header without request (should fail) + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_sh(true, 0); + HELPER_EXPECT_FAILED(controller.on_sequence_header(&video_sh, &fmt)); + + // Set request and try again + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&video_sh, &fmt)); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, Performance_MultipleSegments) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(muxer.on_publish(&req)); + HELPER_ASSERT_SUCCESS(muxer.update_config(&req)); + + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, true)); + + int initial_seq = muxer.sequence_no(); + + // Write many samples to create multiple segments + for (int i = 0; i < 500; i++) { + MockSrsSharedPtrMessage video_msg(true, i * 40); + HELPER_ASSERT_SUCCESS(muxer.write_video(&video_msg, &fmt)); + + if (i % 2 == 0) { // Write audio less frequently + MockSrsSharedPtrMessage audio_msg(false, i * 40); + HELPER_ASSERT_SUCCESS(muxer.write_audio(&audio_msg, &fmt)); + } + } + + // Should have created multiple segments + EXPECT_GT(muxer.sequence_no(), initial_seq); + EXPECT_GT(muxer.duration(), 0); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, Compatibility_SequenceHeaderIgnore) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + + MockSrsFormat fmt; + + // Create audio sequence header message + MockSrsSharedPtrMessage audio_sh(false, 0); + + // Should ignore sequence headers in write_audio + HELPER_ASSERT_SUCCESS(controller.write_audio(&audio_sh, &fmt)); + + // Regular audio message should be processed + MockSrsSharedPtrMessage audio_msg(false, 1000); + HELPER_ASSERT_SUCCESS(controller.write_audio(&audio_msg, &fmt)); + + controller.dispose(); +} diff --git a/trunk/src/utest/srs_utest_fmp4.hpp b/trunk/src/utest/srs_utest_fmp4.hpp new file mode 100644 index 000000000..c19aa4fe9 --- /dev/null +++ b/trunk/src/utest/srs_utest_fmp4.hpp @@ -0,0 +1,15 @@ +// +// Copyright (c) 2013-2025 The SRS Authors +// +// SPDX-License-Identifier: MIT +// + +#ifndef SRS_UTEST_FMP4_HPP +#define SRS_UTEST_FMP4_HPP + +/* +#include +*/ +#include + +#endif diff --git a/trunk/src/utest/srs_utest_http.cpp b/trunk/src/utest/srs_utest_http.cpp index d895c0b95..9c2528bae 100644 --- a/trunk/src/utest/srs_utest_http.cpp +++ b/trunk/src/utest/srs_utest_http.cpp @@ -1569,6 +1569,50 @@ VOID TEST(ProtocolHTTPTest, VodStreamHandlers) HELPER_ASSERT_SUCCESS(h.serve_http(&w2, &r)); __MOCK_HTTP_EXPECT_STREQ(200, "livestream-13.ts?hls_ctx=123456", w2); } + + // fmp4 .m4s with hls_ctx test + if (true) { + SrsHttpMuxEntry e; + e.pattern = "/"; + + SrsVodStream h("/tmp"); + h.set_fs_factory(new MockFileReaderFactory("livestream-13.m4s")); + h.set_path_check(_mock_srs_path_always_exists); + h.entry = &e; + + MockResponseWriter w; + SrsHttpMessage r(NULL, NULL); + HELPER_ASSERT_SUCCESS(r.set_url("/index.m3u8?hls_ctx=123456", false)); + + HELPER_ASSERT_SUCCESS(h.serve_http(&w, &r)); + __MOCK_HTTP_EXPECT_STREQ4(200, "/index.m3u8?hls_ctx=123456\n", w); + + MockResponseWriter w2; + HELPER_ASSERT_SUCCESS(h.serve_http(&w2, &r)); + __MOCK_HTTP_EXPECT_STREQ(200, "livestream-13.m4s?hls_ctx=123456", w2); + } + + // fmp4 init.mp4 with hls_ctx test + if (true) { + SrsHttpMuxEntry e; + e.pattern = "/"; + + SrsVodStream h("/tmp"); + h.set_fs_factory(new MockFileReaderFactory("init.mp4")); + h.set_path_check(_mock_srs_path_always_exists); + h.entry = &e; + + MockResponseWriter w; + SrsHttpMessage r(NULL, NULL); + HELPER_ASSERT_SUCCESS(r.set_url("/index.m3u8?hls_ctx=123456", false)); + + HELPER_ASSERT_SUCCESS(h.serve_http(&w, &r)); + __MOCK_HTTP_EXPECT_STREQ4(200, "/index.m3u8?hls_ctx=123456\n", w); + + MockResponseWriter w2; + HELPER_ASSERT_SUCCESS(h.serve_http(&w2, &r)); + __MOCK_HTTP_EXPECT_STREQ(200, "init.mp4?hls_ctx=123456", w2); + } } VOID TEST(ProtocolHTTPTest, BasicHandlers) diff --git a/trunk/src/utest/srs_utest_mp4.cpp b/trunk/src/utest/srs_utest_mp4.cpp index c4cb8c8f7..295ae1837 100644 --- a/trunk/src/utest/srs_utest_mp4.cpp +++ b/trunk/src/utest/srs_utest_mp4.cpp @@ -12,6 +12,7 @@ using namespace std; #include #include #include +#include VOID TEST(KernelMp4Test, PrintPadding) { @@ -535,6 +536,21 @@ VOID TEST(KernelMp4Test, FullBoxDump) } } +VOID TEST(KernelMp4Test, MOOFBox) +{ + if (true) { + SrsMp4MovieFragmentBox box; + + SrsMp4MovieFragmentHeaderBox *mfhd = new SrsMp4MovieFragmentHeaderBox(); + box.set_mfhd(mfhd); + EXPECT_EQ(box.mfhd(), mfhd); + + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + box.add_traf(traf); + EXPECT_TRUE(traf == box.get(SrsMp4BoxTypeTRAF)); + } +} + VOID TEST(KernelMp4Test, MFHDBox) { srs_error_t err; @@ -898,11 +914,10 @@ VOID TEST(KernelMp4Test, TREXBox) } SrsMp4MovieExtendsBox box; - EXPECT_TRUE(NULL == box.trex()); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - box.set_trex(trex); - EXPECT_TRUE(trex == box.trex()); + box.add_trex(trex); + EXPECT_TRUE(trex == box.get(SrsMp4BoxTypeTREX)); } VOID TEST(KernelMp4Test, TKHDBox) @@ -1823,6 +1838,442 @@ VOID TEST(KernelMp4Test, STSDBox) } } +VOID TEST(KernelMp4Test, SAIZBox) +{ + srs_error_t err; + // flags & 1 == 0; default_sample_info_size == 1 + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + uint8_t data[12+5]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIZ); b.write_1bytes(0); b.write_3bytes(0); + b.write_1bytes(1); b.write_4bytes(0); b.skip(-17); + + HELPER_ASSERT_SUCCESS(saiz.decode(&b)); + EXPECT_EQ(17, (int)saiz.nb_header()); + EXPECT_EQ(0, (int)saiz.version); + EXPECT_EQ(0, (int)saiz.flags); + EXPECT_EQ(1, (int)saiz.default_sample_info_size); + EXPECT_EQ(0, (int)saiz.sample_count); + EXPECT_EQ(0, saiz.sample_info_sizes.size()); + } + + // flags & 1 == 1; default_sample_info_size == 1 + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + uint8_t data[12+13]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIZ); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_1bytes(1); b.write_4bytes(0); b.skip(-25); + + HELPER_ASSERT_SUCCESS(saiz.decode(&b)); + EXPECT_EQ(25, (int)saiz.nb_header()); + EXPECT_EQ(0, (int)saiz.version); + EXPECT_EQ(1, (int)saiz.flags); + EXPECT_EQ(1, (int)saiz.aux_info_type); + EXPECT_EQ(2, (int)saiz.aux_info_type_parameter); + EXPECT_EQ(1, (int)saiz.default_sample_info_size); + EXPECT_EQ(0, (int)saiz.sample_count); + EXPECT_EQ(0, saiz.sample_info_sizes.size()); + } + + // flags & 1 == 1; default_sample_info_size == 0; sample_count = 3; + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + uint8_t data[12+16]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIZ); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_1bytes(0); b.write_4bytes(3); + b.write_1bytes(4); b.write_1bytes(5); b.write_1bytes(6); + b.skip(-28); + + HELPER_ASSERT_SUCCESS(saiz.decode(&b)); + EXPECT_EQ(28, (int)saiz.nb_header()); + EXPECT_EQ(0, (int)saiz.version); + EXPECT_EQ(1, (int)saiz.flags); + EXPECT_EQ(1, (int)saiz.aux_info_type); + EXPECT_EQ(2, (int)saiz.aux_info_type_parameter); + EXPECT_EQ(0, (int)saiz.default_sample_info_size); + EXPECT_EQ(3, (int)saiz.sample_count); + EXPECT_EQ(3, saiz.sample_info_sizes.size()); + EXPECT_EQ(4, saiz.sample_info_sizes[0]); + EXPECT_EQ(5, saiz.sample_info_sizes[1]); + EXPECT_EQ(6, saiz.sample_info_sizes[2]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + saiz.flags = 0; + saiz.default_sample_info_size = 1; + saiz.sample_count = 0; + + EXPECT_EQ(17, saiz.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("default_sample_info_size=1, sample_count=0", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + saiz.flags = 1; + saiz.default_sample_info_size = 1; + saiz.sample_count = 0; + + EXPECT_EQ(25, saiz.nb_header()); + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("default_sample_info_size=1, sample_count=0", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + saiz.flags = 1; + saiz.default_sample_info_size = 0; + saiz.sample_count = 1; + saiz.sample_info_sizes.push_back(4); + + EXPECT_EQ(26, saiz.nb_header()); + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("default_sample_info_size=0, sample_count=1", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SAIOBox) +{ + srs_error_t err; + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + uint8_t data[12+8]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIO); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(1); b.write_4bytes(2); b.skip(-20); + + HELPER_ASSERT_SUCCESS(saio.decode(&b)); + EXPECT_EQ(20, (int)saio.nb_header()); + EXPECT_EQ(0, (int)saio.version); + EXPECT_EQ(0, (int)saio.flags); + EXPECT_EQ(1, (int)saio.offsets.size()); + EXPECT_EQ(2, (int)saio.offsets[0]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + uint8_t data[12+16]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIO); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_4bytes(1); b.write_4bytes(2); b.skip(-28); + + HELPER_ASSERT_SUCCESS(saio.decode(&b)); + EXPECT_EQ(28, (int)saio.nb_header()); + EXPECT_EQ(0, (int)saio.version); + EXPECT_EQ(1, (int)saio.flags); + EXPECT_EQ(1, (int)saio.aux_info_type); + EXPECT_EQ(2, (int)saio.aux_info_type_parameter); + EXPECT_EQ(1, (int)saio.offsets.size()); + EXPECT_EQ(2, (int)saio.offsets[0]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + uint8_t data[12+20]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIO); b.write_1bytes(1); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_4bytes(1); b.write_8bytes(2); b.skip(-32); + + HELPER_ASSERT_SUCCESS(saio.decode(&b)); + EXPECT_EQ(32, (int)saio.nb_header()); + EXPECT_EQ(1, (int)saio.version); + EXPECT_EQ(1, (int)saio.flags); + EXPECT_EQ(1, (int)saio.aux_info_type); + EXPECT_EQ(2, (int)saio.aux_info_type_parameter); + EXPECT_EQ(1, (int)saio.offsets.size()); + EXPECT_EQ(2, (int)saio.offsets[0]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + saio.version = 0; + saio.flags = 0; + saio.offsets.push_back(2); + EXPECT_EQ(20, (int)saio.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("entry_count=1", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + saio.version = 0; + saio.flags = 1; + saio.offsets.push_back(2); + EXPECT_EQ(28, (int)saio.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("entry_count=1", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + saio.version = 1; + saio.flags = 1; + saio.offsets.push_back(2); + EXPECT_EQ(32, (int)saio.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("entry_count=1", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SENCBox) +{ + srs_error_t err; + + if (true) { + SrsMp4SampleEncryptionBox senc(8); + + uint8_t data[12+4]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSENC); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(0); b.skip(-16); + + HELPER_ASSERT_SUCCESS(senc.decode(&b)); + EXPECT_EQ(16, (int)senc.nb_header()); + EXPECT_EQ(0, (int)senc.version); + EXPECT_EQ(0, (int)senc.flags); + EXPECT_EQ(0, (int)senc.entries.size()); + } + + if (true) { + SrsMp4SampleEncryptionBox senc(8); + + uint8_t data[12+12]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSENC); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(1); b.write_8bytes(1); b.skip(-24); + + HELPER_ASSERT_SUCCESS(senc.decode(&b)); + EXPECT_EQ(24, (int)senc.nb_header()); + EXPECT_EQ(0, (int)senc.version); + EXPECT_EQ(0, (int)senc.flags); + EXPECT_EQ(1, (int)senc.entries.size()); + } +} + +VOID TEST(KernelMp4Test, FRMABox) +{ + srs_error_t err; + + if (true) { + SrsMp4OriginalFormatBox frma(1); + uint8_t data[8+4]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(0); b.write_4bytes(SrsMp4BoxTypeFRMA); + b.write_4bytes(1); b.skip(-12); + + HELPER_ASSERT_SUCCESS(frma.decode(&b)); + EXPECT_EQ(12, (int)frma.nb_header()); + } + + if (true) { + SrsMp4OriginalFormatBox frma(1); + EXPECT_EQ(12, (int)frma.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + frma.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("original format=1\n", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SCHMBox) { + srs_error_t err; + + if (true) { + SrsMp4SchemeTypeBox schm; + uint8_t data[12+8]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSCHM); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(1); b.write_4bytes(2); b.skip(-20); + + HELPER_ASSERT_SUCCESS(schm.decode(&b)); + EXPECT_EQ(20, (int)schm.nb_header()); + EXPECT_EQ(0, (int)schm.version); + EXPECT_EQ(0, (int)schm.flags); + EXPECT_EQ(1, (int)schm.scheme_type); + EXPECT_EQ(2, (int)schm.scheme_version); + } + + if (true) { + SrsMp4SchemeTypeBox schm; + uint8_t data[12+8+4]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSCHM); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_1bytes(65); b.write_1bytes(65); b.write_1bytes(65); b.write_1bytes(0); b.skip(-24); + + HELPER_ASSERT_SUCCESS(schm.decode(&b)); + EXPECT_EQ(24, (int)schm.nb_header()); + EXPECT_EQ(0, (int)schm.version); + EXPECT_EQ(1, (int)schm.flags); + EXPECT_EQ(1, (int)schm.scheme_type); + EXPECT_EQ(2, (int)schm.scheme_version); + + stringstream ss; + SrsMp4DumpContext dc; + schm.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("scheme_type=1, scheme_version=2\nscheme_uri=AAA\n", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SrsInitMp4Segment) +{ + srs_error_t err; + // write single video segment + if (true) { + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init.mp4"); + SrsFormat fmt; + HELPER_ASSERT_SUCCESS(fmt.initialize()); + + uint8_t raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + HELPER_ASSERT_SUCCESS(fmt.on_video(0, (char*)raw, sizeof(raw))); + + HELPER_ASSERT_SUCCESS(segment.write_video_only(&fmt, 1)); + EXPECT_TRUE(fw.filesize() > 0); + } + + // write single audio segment + if (true) { + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init.mp4"); + SrsFormat fmt; + HELPER_ASSERT_SUCCESS(fmt.initialize()); + + uint8_t raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + + HELPER_ASSERT_SUCCESS(fmt.on_audio(0, (char*)raw, sizeof(raw))); + + HELPER_ASSERT_SUCCESS(segment.write_audio_only(&fmt, 1)); + EXPECT_TRUE(fw.filesize() > 0); + } + + // write both audio and video segment + if (true) { + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init.mp4"); + SrsFormat fmt; + HELPER_ASSERT_SUCCESS(fmt.initialize()); + + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + + HELPER_ASSERT_SUCCESS(fmt.on_video(0, (char*)video_raw, sizeof(video_raw))); + + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + + HELPER_ASSERT_SUCCESS(fmt.on_audio(0, (char*)audio_raw, sizeof(audio_raw))); + + HELPER_ASSERT_SUCCESS(segment.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); + } +} + +VOID TEST(KernelMp4Test, SrsFmp4SegmentEncoder) +{ + srs_error_t err; + + if (true) { + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + encoder.initialize(&fw, 0, 0, 1, 2); + + SrsFormat video_fmt; + HELPER_ASSERT_SUCCESS(video_fmt.initialize()); + + SrsFormat audio_fmt; + HELPER_ASSERT_SUCCESS(audio_fmt.initialize()); + + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + + HELPER_ASSERT_SUCCESS(video_fmt.on_video(0, (char*)video_raw, sizeof(video_raw))); + + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + + HELPER_ASSERT_SUCCESS(audio_fmt.on_audio(0, (char*)audio_raw, sizeof(audio_raw))); + + SrsVideoAvcFrameType video_frame_type = video_fmt.video->frame_type; + uint32_t cts = (uint32_t)video_fmt.video->cts; + + uint32_t dts = 0; + uint32_t pts = dts + cts; + + uint8_t* video_sample = (uint8_t*)video_fmt.raw; + uint32_t nb_video_sample = (uint32_t)video_fmt.nb_raw; + encoder.write_sample(SrsMp4HandlerTypeVIDE, video_frame_type, dts, pts, video_sample, nb_video_sample); + uint8_t* audio_sample = (uint8_t*)audio_fmt.raw; + uint32_t nb_audio_sample = (uint32_t)audio_fmt.nb_raw; + encoder.write_sample(SrsMp4HandlerTypeSOUN, 0, 0, 0, audio_sample, nb_audio_sample); + encoder.flush(dts); + EXPECT_TRUE(fw.filesize() > 0); + } +} + VOID TEST(KernelMp4Test, SrsMp4M2tsInitEncoder) { srs_error_t err; @@ -1867,5 +2318,32 @@ VOID TEST(KernelMp4Test, SrsMp4M2tsInitEncoder) HELPER_ASSERT_SUCCESS(enc.write(&fmt, false, 1)); EXPECT_TRUE(fw.filesize() > 0); } + + if (true) { + MockSrsFileWriter fw; + HELPER_ASSERT_SUCCESS(fw.open("test.mp4")); + + SrsMp4M2tsInitEncoder enc; + HELPER_ASSERT_SUCCESS(enc.initialize(&fw)); + + SrsFormat fmt; + EXPECT_TRUE(srs_success == fmt.initialize()); + + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + EXPECT_TRUE(srs_success == fmt.on_video(0, (char*)video_raw, sizeof(video_raw))); + + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + EXPECT_TRUE(srs_success == fmt.on_audio(0, (char*)audio_raw, sizeof(audio_raw))); + + HELPER_ASSERT_SUCCESS(enc.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); + } }