diff --git a/README.md b/README.md index 2348e00b8..ce2410922 100755 --- a/README.md +++ b/README.md @@ -3,14 +3,9 @@ ![](http://ossrs.net/gif/v1/sls.gif?site=github.com&path=/srs/develop) [![](https://github.com/ossrs/srs/actions/workflows/codeql-analysis.yml/badge.svg?branch=develop)](https://github.com/ossrs/srs/actions?query=workflow%3ACodeQL+branch%3Adevelop) [![](https://github.com/ossrs/srs/actions/workflows/release.yml/badge.svg)](https://github.com/ossrs/srs/actions/workflows/release.yml?query=workflow%3ARelease) -[![](https://github.com/ossrs/srs/actions/workflows/test.yml/badge.svg?branch=develop)](https://github.com/ossrs/srs/actions?query=workflow%3ATest+branch%3Adevelop) -[![](https://codecov.io/gh/ossrs/srs/branch/develop/graph/badge.svg?token=Zx2LhdtA39)](https://app.codecov.io/gh/ossrs/srs/tree/develop) -[![](https://ossrs.net/wiki/images/wechat-badge4.svg)](https://ossrs.net/lts/zh-cn/contact#discussion) [![](https://img.shields.io/twitter/follow/srs_server?style=social)](https://twitter.com/srs_server) [![](https://img.shields.io/badge/SRS-YouTube-red)](https://www.youtube.com/@srs_server) [![](https://badgen.net/discord/members/yZ4BnPmHAd)](https://discord.gg/yZ4BnPmHAd) -[![](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fossrs%2Fsrs.svg?type=small)](https://app.fossa.com/projects/git%2Bgithub.com%2Fossrs%2Fsrs?ref=badge_small) -[![](https://badgen.net/badge/srs/stackoverflow/orange?icon=terminal)](https://stackoverflow.com/questions/tagged/simple-realtime-server) [![](https://opencollective.com/srs-server/tiers/badge.svg)](https://opencollective.com/srs-server) [![](https://img.shields.io/docker/pulls/ossrs/srs)](https://hub.docker.com/r/ossrs/srs/tags) diff --git a/trunk/3rdparty/srs-bench/blackbox/hls_test.go b/trunk/3rdparty/srs-bench/blackbox/hls_test.go index 5b43e9a7d..395bf6da3 100644 --- a/trunk/3rdparty/srs-bench/blackbox/hls_test.go +++ b/trunk/3rdparty/srs-bench/blackbox/hls_test.go @@ -114,3 +114,86 @@ func TestFast_RtmpPublish_HlsPlay_Basic(t *testing.T) { } } } + +func TestFast_RtmpPublish_HlsPlay_Fmp4(t *testing.T) { + // This case is run in parallel. + t.Parallel() + + // Setup the max timeout for this case. + ctx, cancel := context.WithTimeout(logger.WithContext(context.Background()), time.Duration(*srsTimeout)*time.Millisecond) + defer cancel() + + // Check a set of errors. + var r0, r1, r2, r3, r4 error + defer func(ctx context.Context) { + if err := filterTestError(ctx.Err(), r0, r1, r2, r3, r4); err != nil { + t.Errorf("Fail for err %+v", err) + } else { + logger.Tf(ctx, "test done with err %+v", err) + } + }(ctx) + + var wg sync.WaitGroup + defer wg.Wait() + + // Start SRS server and wait for it to be ready. + svr := NewSRSServer(func(v *srsServer) { + v.envs = []string{ + "SRS_HTTP_SERVER_ENABLED=on", + "SRS_VHOST_HLS_ENABLED=on", + "SRS_VHOST_HLS_HLS_USE_FMP4=on", + } + }) + wg.Add(1) + go func() { + defer wg.Done() + r0 = svr.Run(ctx, cancel) + }() + + // Start FFmpeg to publish stream. + streamID := fmt.Sprintf("stream-%v-%v", os.Getpid(), rand.Int()) + streamURL := fmt.Sprintf("rtmp://localhost:%v/live/%v", svr.RTMPPort(), streamID) + ffmpeg := NewFFmpeg(func(v *ffmpegClient) { + v.args = []string{ + "-stream_loop", "-1", "-re", "-i", *srsPublishAvatar, "-c", "copy", "-f", "flv", streamURL, + } + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r1 = ffmpeg.Run(ctx, cancel) + }() + + // Start FFprobe to detect and verify stream. + duration := time.Duration(*srsFFprobeDuration) * time.Millisecond + ffprobe := NewFFprobe(func(v *ffprobeClient) { + v.dvrFile = path.Join(svr.WorkDir(), "objs", fmt.Sprintf("srs-ffprobe-%v.mp4", streamID)) + v.streamURL = fmt.Sprintf("http://localhost:%v/live/%v.m3u8", svr.HTTPPort(), streamID) + v.duration, v.timeout = duration, time.Duration(*srsFFprobeTimeout)*time.Millisecond + }) + wg.Add(1) + go func() { + defer wg.Done() + <-svr.ReadyCtx().Done() + r2 = ffprobe.Run(ctx, cancel) + }() + + // Fast quit for probe done. + select { + case <-ctx.Done(): + case <-ffprobe.ProbeDoneCtx().Done(): + defer cancel() + + str, m := ffprobe.Result() + if len(m.Streams) != 2 { + r3 = errors.Errorf("invalid streams=%v, %v, %v", len(m.Streams), m.String(), str) + } + + // Note that HLS score is low, so we only check duration. Note that only check half of duration, because we + // might get only some pieces of segments. + if dv := m.Duration(); dv < duration/2 { + r4 = errors.Errorf("short duration=%v < %v, %v, %v", dv, duration/2, m.String(), str) + } + } +} diff --git a/trunk/3rdparty/srs-bench/blackbox/util.go b/trunk/3rdparty/srs-bench/blackbox/util.go index ff9d91424..53a6e9d29 100644 --- a/trunk/3rdparty/srs-bench/blackbox/util.go +++ b/trunk/3rdparty/srs-bench/blackbox/util.go @@ -568,6 +568,8 @@ func (v *srsServer) Run(ctx context.Context, cancel context.CancelFunc) error { "SRS_VHOST_HLS_HLS_PATH=./objs/nginx/html", "SRS_VHOST_HLS_HLS_M3U8_FILE=[app]/[stream].m3u8", "SRS_VHOST_HLS_HLS_TS_FILE=[app]/[stream]-[seq].ts", + "SRS_VHOST_HLS_HLS_FMP4_FILE=[app]/[stream]-[seq].m4s", + "SRS_VHOST_HLS_HLS_INIT_FILE=[app]/[stream]-init.mp4", }...) // For variables. v.process.env = append(v.process.env, []string{ diff --git a/trunk/conf/full.conf b/trunk/conf/full.conf index cd7df4e7f..a38286f92 100644 --- a/trunk/conf/full.conf +++ b/trunk/conf/full.conf @@ -1821,6 +1821,13 @@ vhost hls.srs.com { # default: off enabled on; + # whether to use fmp4 as container + # The default value is off, then HLS use ts as container format, + # if on, HLS use fmp4 as container format. + # Overwrite by env SRS_VHOST_HLS_HLS_USE_FMP4 for all vhosts. + # default: off + hls_use_fmp4 on; + # the hls fragment in seconds, the duration of a piece of ts. # Overwrite by env SRS_VHOST_HLS_HLS_FRAGMENT for all vhosts. # default: 10 @@ -1886,6 +1893,44 @@ vhost hls.srs.com { # Overwrite by env SRS_VHOST_HLS_HLS_TS_FILE for all vhosts. # default: [app]/[stream]-[seq].ts hls_ts_file [app]/[stream]-[seq].ts; + # the hls fmp4 file name. + # we supports some variables to generate the filename. + # [vhost], the vhost of stream. + # [app], the app of stream. + # [stream], the stream name of stream. + # [2006], replace this const to current year. + # [01], replace this const to current month. + # [02], replace this const to current date. + # [15], replace this const to current hour. + # [04], replace this const to current minute. + # [05], replace this const to current second.p + # [999], replace this const to current millisecond. + # [timestamp],replace this const to current UNIX timestamp in ms. + # [seq], the sequence number of fmp4. + # [duration], replace this const to current ts duration. + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/dvr#custom-path + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/delivery-hls#hls-config + # Overwrite by env SRS_VHOST_HLS_HLS_FMP4_FILE for all vhosts. + # default: [app]/[stream]-[seq].m4s + hls_fmp4_file [app]/[stream]-[seq].m4s; + # the hls init mp4 file name. + # we supports some variables to generate the filename. + # [vhost], the vhost of stream. + # [app], the app of stream. + # [stream], the stream name of stream. + # [2006], replace this const to current year. + # [01], replace this const to current month. + # [02], replace this const to current date. + # [15], replace this const to current hour. + # [04], replace this const to current minute. + # [05], replace this const to current second. + # [999], replace this const to current millisecond. + # [timestamp],replace this const to current UNIX timestamp in ms. + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/dvr#custom-path + # @see https://ossrs.net/lts/zh-cn/docs/v4/doc/delivery-hls#hls-config + # Overwrite by env SRS_VHOST_HLS_HLS_INIT_FILE for all vhosts. + # default: [app]/[stream]/init.mp4 + hls_init_file [app]/[stream]/init.mp4; # the hls entry prefix, which is base url of ts url. # for example, the prefix is: # http://your-server/ diff --git a/trunk/conf/hls.mp4.conf b/trunk/conf/hls.mp4.conf new file mode 100644 index 000000000..0079ba688 --- /dev/null +++ b/trunk/conf/hls.mp4.conf @@ -0,0 +1,29 @@ +# the config for srs to delivery hls +# @see https://ossrs.net/lts/zh-cn/docs/v4/doc/sample-hls +# @see full.conf for detail config. + +listen 1935; +max_connections 1000; +daemon off; +srs_log_tank console; +http_server { + enabled on; + listen 8080; + dir ./objs/nginx/html; +} +http_api { + enabled on; + listen 1985; +} +vhost __defaultVhost__ { + hls { + enabled on; + hls_use_fmp4 on; + hls_path ./objs/nginx/html; + hls_fragment 10; + hls_window 60; + hls_m3u8_file [app]/[stream].m3u8; + hls_init_file [app]/[stream]-init.mp4; + hls_fmp4_file [app]/[stream]-[seq].m4s; + } +} diff --git a/trunk/configure b/trunk/configure index fdc909631..5c22f8916 100755 --- a/trunk/configure +++ b/trunk/configure @@ -471,7 +471,7 @@ if [[ $SRS_UTEST == YES ]]; then "srs_utest_config" "srs_utest_rtmp" "srs_utest_http" "srs_utest_avc" "srs_utest_reload" "srs_utest_mp4" "srs_utest_service" "srs_utest_app" "srs_utest_rtc" "srs_utest_config2" "srs_utest_protocol" "srs_utest_protocol2" "srs_utest_kernel2" "srs_utest_protocol3" - "srs_utest_st" "srs_utest_rtc2" "srs_utest_rtc3") + "srs_utest_st" "srs_utest_rtc2" "srs_utest_rtc3" "srs_utest_fmp4") if [[ $SRS_SRT == YES ]]; then MODULE_FILES+=("srs_utest_srt") fi diff --git a/trunk/doc/CHANGELOG.md b/trunk/doc/CHANGELOG.md index 96cd485bb..077c4a959 100644 --- a/trunk/doc/CHANGELOG.md +++ b/trunk/doc/CHANGELOG.md @@ -7,6 +7,7 @@ The changelog for SRS. ## SRS 7.0 Changelog +* v7.0, 2025-08-11, Merge [#4159](https://github.com/ossrs/srs/pull/4159): Feature: Support HLS with fmp4 segment for HEVC/LLHLS. v7.0.51 (#4159) * v7.0, 2025-08-11, Merge [#4432](https://github.com/ossrs/srs/pull/4432): AI: HTTP-FLV: Fix heap-use-after-free crash during stream unmount. v7.0.50 (#4432) * v7.0, 2025-07-28, Merge [#4245](https://github.com/ossrs/srs/pull/4245): Allow Forward to be configured with Env Var. v7.0.49 (#4245) * v7.0, 2025-07-16, Merge [#4295](https://github.com/ossrs/srs/pull/4295): RTC: audio packet jitter buffer. v7.0.48 (#4295) diff --git a/trunk/src/app/srs_app_config.cpp b/trunk/src/app/srs_app_config.cpp index 3590d0dba..d67ab5a56 100644 --- a/trunk/src/app/srs_app_config.cpp +++ b/trunk/src/app/srs_app_config.cpp @@ -2693,7 +2693,7 @@ srs_error_t SrsConfig::check_normal_config() && m != "hls_storage" && m != "hls_mount" && m != "hls_td_ratio" && m != "hls_aof_ratio" && m != "hls_acodec" && m != "hls_vcodec" && m != "hls_m3u8_file" && m != "hls_ts_file" && m != "hls_ts_floor" && m != "hls_cleanup" && m != "hls_nb_notify" && m != "hls_wait_keyframe" && m != "hls_dispose" && m != "hls_keys" && m != "hls_fragments_per_key" && m != "hls_key_file" - && m != "hls_key_file_path" && m != "hls_key_url" && m != "hls_dts_directly" && m != "hls_ctx" && m != "hls_ts_ctx") { + && m != "hls_key_file_path" && m != "hls_key_url" && m != "hls_dts_directly" && m != "hls_ctx" && m != "hls_ts_ctx" && m != "hls_use_fmp4" && m != "hls_fmp4_file" && m != "hls_init_file") { return srs_error_new(ERROR_SYSTEM_CONFIG_INVALID, "illegal vhost.hls.%s of %s", m.c_str(), vhost->arg0().c_str()); } @@ -7051,6 +7051,31 @@ bool SrsConfig::get_hls_enabled(SrsConfDirective* vhost) return SRS_CONF_PREFER_FALSE(conf->arg0()); } +bool SrsConfig::get_hls_use_fmp4(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.hls.hls_use_fmp4"); // SRS_VHOST_HLS_HLS_USE_FMP4 + + static bool DEFAULT = false; + + SrsConfDirective* conf = get_vhost(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls"); + + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_use_fmp4"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return SRS_CONF_PREFER_FALSE(conf->arg0()); +} + string SrsConfig::get_hls_entry_prefix(string vhost) { SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_entry_prefix"); // SRS_VHOST_HLS_HLS_ENTRY_PREFIX @@ -7127,6 +7152,44 @@ string SrsConfig::get_hls_ts_file(string vhost) return conf->arg0(); } +string SrsConfig::get_hls_fmp4_file(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_fmp4_file"); // SRS_VHOST_HLS_HLS_FMP4_FILE + + static string DEFAULT = "[app]/[stream]-[seq].m4s"; + + SrsConfDirective* conf = get_hls(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_fmp4_file"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return conf->arg0(); +} + +string SrsConfig::get_hls_init_file(std::string vhost) +{ + SRS_OVERWRITE_BY_ENV_STRING("srs.vhost.hls.hls_init_file"); // SRS_VHOST_HLS_HLS_INIT_FILE + + static string DEFAULT = "[app]/[stream]/init.mp4"; + + SrsConfDirective* conf = get_hls(vhost); + if (!conf) { + return DEFAULT; + } + + conf = conf->get("hls_init_file"); + if (!conf || conf->arg0().empty()) { + return DEFAULT; + } + + return conf->arg0(); +} + bool SrsConfig::get_hls_ts_floor(string vhost) { SRS_OVERWRITE_BY_ENV_BOOL("srs.vhost.hls.hls_ts_floor"); // SRS_VHOST_HLS_HLS_TS_FLOOR diff --git a/trunk/src/app/srs_app_config.hpp b/trunk/src/app/srs_app_config.hpp index 74e7617a0..1085faeb9 100644 --- a/trunk/src/app/srs_app_config.hpp +++ b/trunk/src/app/srs_app_config.hpp @@ -943,6 +943,8 @@ public: // Whether HLS is enabled. virtual bool get_hls_enabled(std::string vhost); virtual bool get_hls_enabled(SrsConfDirective* vhost); + // Whether HLS use fmp4 container format + virtual bool get_hls_use_fmp4(std::string vhost); // Get the HLS m3u8 list ts segment entry prefix info. virtual std::string get_hls_entry_prefix(std::string vhost); // Get the HLS ts/m3u8 file store path. @@ -951,6 +953,10 @@ public: virtual std::string get_hls_m3u8_file(std::string vhost); // Get the HLS ts file path template. virtual std::string get_hls_ts_file(std::string vhost); + // Get the HLS fmp4 file path template. + virtual std::string get_hls_fmp4_file(std::string vhost); + // Get the HLS init mp4 file path template. + virtual std::string get_hls_init_file(std::string vhost); // Whether enable the floor(timestamp/hls_fragment) for variable timestamp. virtual bool get_hls_ts_floor(std::string vhost); // Get the hls fragment time, in srs_utime_t. @@ -995,6 +1001,7 @@ public: // Whether enable hls_ctx virtual bool get_hls_ctx_enabled(std::string vhost); // Whether enable session for ts file. + // The ts file including .ts file for MPEG-ts segment, .m4s file and init.mp4 file for fmp4 segment. virtual bool get_hls_ts_ctx_enabled(std::string vhost); // hds section private: diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 1a4f9f7f7..4c7c6759d 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -31,6 +31,7 @@ using namespace std; #include #include #include +#include #include #include #include @@ -76,6 +77,186 @@ srs_error_t SrsHlsSegment::rename() return SrsFragment::rename(); } +SrsInitMp4Segment::SrsInitMp4Segment(SrsFileWriter* fw) +{ + fw_ = fw; + const_iv_size_ = 0; +} + +SrsInitMp4Segment::~SrsInitMp4Segment() +{ + fw_->close(); +} + +srs_error_t SrsInitMp4Segment::config_cipher(unsigned char* kid, unsigned char* const_iv, uint8_t const_iv_size) +{ + if (const_iv_size != 8 && const_iv_size != 16) { + return srs_error_new(ERROR_MP4_BOX_STRING, "invalidate const_iv_size=%d", const_iv_size); + } + + memcpy(kid_, kid, 16); + memcpy(const_iv_, const_iv, const_iv_size); + const_iv_size_ = const_iv_size; + + // CBCS encryption: For example, 1 encrypt block, 9 skip blocks (10% encryption) + init_.config_encryption(1, 9, kid_, const_iv, const_iv_size); + + return srs_success; +} + +srs_error_t SrsInitMp4Segment::write(SrsFormat* format, int v_tid, int a_tid) +{ + srs_error_t err = srs_success; + + if ((err = init_encoder()) != srs_success) { + return srs_error_wrap(err, "init encoder"); + } + + if ((err = init_.write(format, v_tid, a_tid)) != srs_success) { + return srs_error_wrap(err, "write init"); + } + + return err; +} + +srs_error_t SrsInitMp4Segment::write_video_only(SrsFormat* format, int v_tid) +{ + srs_error_t err = srs_success; + + if ((err = init_encoder()) != srs_success) { + return srs_error_wrap(err, "init encoder"); + } + + if ((err = init_.write(format, true, v_tid)) != srs_success) { + return srs_error_wrap(err, "write init"); + } + + return err; +} + +srs_error_t SrsInitMp4Segment::write_audio_only(SrsFormat* format, int a_tid) +{ + srs_error_t err = srs_success; + + if ((err = init_encoder()) != srs_success) { + return srs_error_wrap(err, "init encoder"); + } + + if ((err = init_.write(format, false, a_tid)) != srs_success) { + return srs_error_wrap(err, "write init"); + } + + return err; +} + +srs_error_t SrsInitMp4Segment::init_encoder() +{ + srs_error_t err = srs_success; + + srs_assert(!fullpath().empty()); + + string path_tmp = tmppath(); + if ((err = fw_->open(path_tmp)) != srs_success) { + return srs_error_wrap(err, "Open init mp4 failed, path=%s", path_tmp.c_str()); + } + + if ((err = init_.initialize(fw_)) != srs_success) { + return srs_error_wrap(err, "init"); + } + + return err; +} + +SrsHlsM4sSegment::SrsHlsM4sSegment(SrsFileWriter* fw) +{ + fw_ = fw; +} + +SrsHlsM4sSegment::~SrsHlsM4sSegment() +{ +} + +srs_error_t SrsHlsM4sSegment::initialize(int64_t time, uint32_t v_tid, uint32_t a_tid, int sequence_number, std::string m4s_path) +{ + srs_error_t err = srs_success; + + set_path(m4s_path); + + set_number(sequence_number); + if ((err = create_dir()) != srs_success) { + return srs_error_wrap(err, "create hls m4s segment dir."); + } + + if ((err = fw_->open(tmppath())) != srs_success) { + return srs_error_wrap(err, "open hls m4s segment tmp file."); + } + + if ((err = enc_.initialize(fw_, sequence_number, time, v_tid, a_tid)) != srs_success) + { + return srs_error_wrap(err, "initialize SrsFmp4SegmentEncoder"); + } + + return err; +} + +void SrsHlsM4sSegment::config_cipher(unsigned char* key, unsigned char* iv) +{ + // TODO: set key and iv to mp4 box + enc_.config_cipher(key, iv); + memcpy(this->iv, iv, 16); +} + +srs_error_t SrsHlsM4sSegment::write(SrsSharedPtrMessage* shared_msg, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (shared_msg->is_audio()) { + uint8_t* sample = (uint8_t*)format->raw; + uint32_t nb_sample = (uint32_t)format->nb_raw; + + uint32_t dts = (uint32_t)shared_msg->timestamp; + if ((err = enc_.write_sample(SrsMp4HandlerTypeSOUN, 0x00, dts, dts, sample, nb_sample)) != srs_success) { + return srs_error_wrap(err, "m4s segment write audio sample"); + } + } else if (shared_msg->is_video()) { + SrsVideoAvcFrameType frame_type = format->video->frame_type; + uint32_t cts = (uint32_t)format->video->cts; + + uint32_t dts = (uint32_t)shared_msg->timestamp; + uint32_t pts = dts + cts; + + uint8_t* sample = (uint8_t*)format->raw; + uint32_t nb_sample = (uint32_t)format->nb_raw; + if ((err = enc_.write_sample(SrsMp4HandlerTypeVIDE, frame_type, dts, pts, sample, nb_sample)) != srs_success) { + return srs_error_wrap(err, "m4s segment write video sample"); + } + } else { + srs_trace("the sample m4s segment write is neither video nor audio sample."); + return err; + } + + append(shared_msg->timestamp); + + return err; +} + +srs_error_t SrsHlsM4sSegment::reap(uint64_t dts) +{ + srs_error_t err = srs_success; + + if ((err = enc_.flush(dts)) != srs_success) { + return srs_error_wrap(err, "m4s flush encoder."); + } + + fw_->close(); + + if ((err = rename()) != srs_success) { + return srs_error_wrap(err, "m4s segment rename."); + } + + return err; +} + SrsDvrAsyncCallOnHls::SrsDvrAsyncCallOnHls(SrsContextId c, SrsRequest* r, string p, string t, string m, string mu, int s, srs_utime_t d) { req = r->copy(); @@ -182,6 +363,695 @@ string SrsDvrAsyncCallOnHlsNotify::to_string() return "on_hls_notify: " + ts_url; } +SrsHlsFmp4Muxer::SrsHlsFmp4Muxer() +{ + req_ = NULL; + hls_fragment_ = hls_window_ = 0; + hls_aof_ratio_ = 1.0; + deviation_ts_ = 0; + hls_cleanup_ = true; + hls_wait_keyframe_ = true; + previous_floor_ts_ = 0; + accept_floor_ts_ = 0; + hls_ts_floor_ = false; + max_td_ = 0; + writer_ = NULL; + sequence_no_ = 0; + current_ = NULL; + hls_keys_ = false; + hls_fragments_per_key_ = 0; + async_ = new SrsAsyncCallWorker(); + segments_ = new SrsFragmentWindow(); + latest_acodec_ = SrsAudioCodecIdForbidden; + latest_vcodec_ = SrsVideoCodecIdForbidden; + video_track_id_ = 0; + audio_track_id_ = 0; + init_mp4_ready_ = false; + video_dts_ = 0; + + memset(key_, 0, 16); + memset(iv_, 0, 16); +} + +SrsHlsFmp4Muxer::~SrsHlsFmp4Muxer() +{ + srs_freep(segments_); + srs_freep(current_); + srs_freep(req_); + srs_freep(async_); + srs_freep(writer_); +} + +void SrsHlsFmp4Muxer::dispose() +{ + srs_error_t err = srs_success; + + segments_->dispose(); + + if (current_) { + if ((err = current_->unlink_tmpfile()) != srs_success) { + srs_warn("Unlink tmp ts failed %s", srs_error_desc(err).c_str()); + srs_freep(err); + } + srs_freep(current_); + } + + if (unlink(m3u8_.c_str()) < 0) { + srs_warn("dispose unlink path failed. file=%s", m3u8_.c_str()); + } + + srs_trace("gracefully dispose hls %s", req_ ? req_->get_stream_url().c_str() : ""); +} + +int SrsHlsFmp4Muxer::sequence_no() +{ + return sequence_no_; +} + +std::string SrsHlsFmp4Muxer::m4s_url() +{ + return current_ ? current_->uri : ""; +} + +srs_utime_t SrsHlsFmp4Muxer::duration() +{ + return current_ ? current_->duration() : 0; +} + +int SrsHlsFmp4Muxer::deviation() +{ + // no floor, no deviation. + if (!hls_ts_floor_) { + return 0; + } + + return deviation_ts_; +} + +SrsAudioCodecId SrsHlsFmp4Muxer::latest_acodec() +{ + return latest_acodec_; +} + +void SrsHlsFmp4Muxer::set_latest_acodec(SrsAudioCodecId v) +{ + latest_acodec_ = v; +} + +SrsVideoCodecId SrsHlsFmp4Muxer::latest_vcodec() +{ + return latest_vcodec_; +} + +void SrsHlsFmp4Muxer::set_latest_vcodec(SrsVideoCodecId v) +{ + latest_vcodec_ = v; +} + +srs_error_t SrsHlsFmp4Muxer::initialize(int v_tid, int a_tid) +{ + video_track_id_ = v_tid; + audio_track_id_ = a_tid; + + return srs_success; +} + +srs_error_t SrsHlsFmp4Muxer::on_publish(SrsRequest* req) +{ + srs_error_t err = srs_success; + + if ((err = async_->start()) != srs_success) { + return srs_error_wrap(err, "async start"); + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_init_mp4(SrsFormat* format, bool has_video, bool has_audio) +{ + srs_error_t err = srs_success; + + std::string vhost = req_->vhost; + std::string stream = req_->stream; + std::string app = req_->app; + + // Get init.mp4 file template from configuration + std::string init_file = _srs_config->get_hls_init_file(vhost); + init_file = srs_path_build_stream(init_file, vhost, app, stream); + + std::string hls_path = _srs_config->get_hls_path(vhost); + std::string path = hls_path + "/" + init_file; + + // Create directory for the init file + std::string init_dir = srs_path_dirname(path); + if ((err = srs_create_dir_recursively(init_dir)) != srs_success) { + return srs_error_wrap(err, "Create init mp4 dir failed, dir=%s", init_dir.c_str()); + } + + SrsUniquePtr init_mp4(new SrsInitMp4Segment(writer_)); + + init_mp4->set_path(path); + + if (hls_keys_) { + init_mp4->config_cipher(kid_, iv_, 16); + } + + if (has_video && has_audio) { + if ((err = init_mp4->write(format, video_track_id_, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "write hls init.mp4 with audio and video"); + } + } else if (has_video) { + if ((err = init_mp4->write_video_only(format, video_track_id_)) != srs_success) { + return srs_error_wrap(err, "write hls init.mp4 with video only"); + } + } else if (has_audio) { + if ((err = init_mp4->write_audio_only(format, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "write hls init.mp4 with audio only"); + } + } else { + return srs_error_new(ERROR_HLS_WRITE_FAILED, "no video and no audio sequence header"); + } + + if ((err = init_mp4->rename()) != srs_success) { + return srs_error_wrap(err, "rename hls init.mp4"); + } + + // the ts url, relative or absolute url. + // TODO: FIXME: Use url and path manager. + std::string mp4_path = init_mp4->fullpath(); + if (srs_string_starts_with(mp4_path, m3u8_dir_)) { + mp4_path = mp4_path.substr(m3u8_dir_.length()); + } + while (srs_string_starts_with(mp4_path, "/")) { + mp4_path = mp4_path.substr(1); + } + + string init_mp4_uri = hls_entry_prefix_; + if (!hls_entry_prefix_.empty() && !srs_string_ends_with(hls_entry_prefix_, "/")) { + init_mp4_uri += "/"; + + // add the http dir to uri. + string http_dir = srs_path_dirname(m3u8_url_); + if (!http_dir.empty()) { + init_mp4_uri += http_dir + "/"; + } + } + init_mp4_uri += mp4_path; + + // Convert to relative URI for m3u8 playlist. + // TODO: Need to resolve the relative URI from m3u8 and init file. + init_mp4_uri_ = srs_path_basename(init_file); + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHls(_srs_context->get_id(), req_, init_mp4->fullpath(), + init_mp4_uri, m3u8_, m3u8_url_, 0, 0))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHlsNotify(_srs_context->get_id(), req_, init_mp4_uri))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + init_mp4_ready_ = true; + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (!current_) { + if ((err = segment_open(shared_audio->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + if (current_->duration() >= hls_fragment_) { + if ((err = segment_close()) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + if ((err = segment_open(shared_audio->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + current_->write(shared_audio, format); + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) +{ + srs_error_t err = srs_success; + + video_dts_ = shared_video->timestamp; + + if (!current_) { + if ((err = segment_open(shared_video->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + bool reopen = current_->duration() >= hls_fragment_; + if (reopen) { + if ((err = segment_close()) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + if ((err = segment_open(shared_video->timestamp * SRS_UTIME_MILLISECONDS)) != srs_success) { + return srs_error_wrap(err, "open segment"); + } + } + + current_->write(shared_video, format); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::on_unpublish() +{ + async_->stop(); + return srs_success; +} + +srs_error_t SrsHlsFmp4Muxer::update_config(SrsRequest* r) +{ + srs_error_t err = srs_success; + + srs_freep(req_); + req_ = r->copy(); + + std::string vhost = req_->vhost; + std::string stream = req_->stream; + std::string app = req_->app; + + hls_fragment_ = _srs_config->get_hls_fragment(vhost); + double hls_td_ratio = _srs_config->get_hls_td_ratio(vhost); + hls_window_ = _srs_config->get_hls_window(vhost); + + // get the hls m3u8 ts list entry prefix config + hls_entry_prefix_ = _srs_config->get_hls_entry_prefix(vhost); + // get the hls path config + hls_path_ = _srs_config->get_hls_path(vhost); + m3u8_url_ = _srs_config->get_hls_m3u8_file(vhost); + hls_m4s_file_ = _srs_config->get_hls_fmp4_file(vhost); + hls_cleanup_ = _srs_config->get_hls_cleanup(vhost); + hls_wait_keyframe_ = _srs_config->get_hls_wait_keyframe(vhost); + // the audio overflow, for pure audio to reap segment. + hls_aof_ratio_ = _srs_config->get_hls_aof_ratio(vhost); + // whether use floor(timestamp/hls_fragment) for variable timestamp + hls_ts_floor_ = _srs_config->get_hls_ts_floor(vhost); + + hls_keys_ = _srs_config->get_hls_keys(vhost); + hls_fragments_per_key_ = _srs_config->get_hls_fragments_per_key(vhost); + hls_key_file_ = _srs_config->get_hls_key_file(vhost); + hls_key_file_path_ = _srs_config->get_hls_key_file_path(vhost); + hls_key_url_ = _srs_config->get_hls_key_url(vhost); + + previous_floor_ts_ = 0; + accept_floor_ts_ = 0; + deviation_ts_ = 0; + + // generate the m3u8 dir and path. + m3u8_url_ = srs_path_build_stream(m3u8_url_, vhost, app, stream); + m3u8_ = hls_path_ + "/" + m3u8_url_; + + // when update config, reset the history target duration. + max_td_ = hls_fragment_ * hls_td_ratio; + + // create m3u8 dir once. + m3u8_dir_ = srs_path_dirname(m3u8_); + if ((err = srs_create_dir_recursively(m3u8_dir_)) != srs_success) { + return srs_error_wrap(err, "create dir"); + } + + if (hls_keys_ && (hls_path_ != hls_key_file_path_)) { + string key_file = srs_path_build_stream(hls_key_file_, vhost, app, stream); + string key_url = hls_key_file_path_ + "/" + key_file; + string key_dir = srs_path_dirname(key_url); + if ((err = srs_create_dir_recursively(key_dir)) != srs_success) { + return srs_error_wrap(err, "create dir"); + } + } + + writer_ = new SrsFileWriter(); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::segment_open(srs_utime_t basetime) +{ + srs_error_t err = srs_success; + + if (current_) { + srs_warn("ignore the segment open, for segment is already open."); + return err; + } + + // new segment. + current_ = new SrsHlsM4sSegment(writer_); + current_->sequence_no = sequence_no_++; + + if ((err = write_hls_key()) != srs_success) { + return srs_error_wrap(err, "write hls key"); + } + + // generate filename. + std::string m4s_file = hls_m4s_file_; + m4s_file = srs_path_build_stream(m4s_file, req_->vhost, req_->app, req_->stream); + if (hls_ts_floor_) { + // accept the floor ts for the first piece. + int64_t current_floor_ts = srs_update_system_time() / hls_fragment_; + if (!accept_floor_ts_) { + accept_floor_ts_ = current_floor_ts - 1; + } else { + accept_floor_ts_++; + } + + // jump when deviation more than 10p + if (accept_floor_ts_ - current_floor_ts > SRS_JUMP_WHEN_PIECE_DEVIATION) { + srs_warn("hls: jmp for ts deviation, current=%" PRId64 ", accept=%" PRId64, current_floor_ts, accept_floor_ts_); + accept_floor_ts_ = current_floor_ts - 1; + } + + // when reap ts, adjust the deviation. + deviation_ts_ = (int)(accept_floor_ts_ - current_floor_ts); + + // dup/jmp detect for ts in floor mode. + if (previous_floor_ts_ && previous_floor_ts_ != current_floor_ts - 1) { + srs_warn("hls: dup/jmp ts, previous=%" PRId64 ", current=%" PRId64 ", accept=%" PRId64 ", deviation=%d", + previous_floor_ts_, current_floor_ts, accept_floor_ts_, deviation_ts_); + } + previous_floor_ts_ = current_floor_ts; + + // we always ensure the piece is increase one by one. + std::stringstream ts_floor; + ts_floor << accept_floor_ts_; + m4s_file = srs_string_replace(m4s_file, "[timestamp]", ts_floor.str()); + + // TODO: FIMXE: we must use the accept ts floor time to generate the hour variable. + m4s_file = srs_path_build_timestamp(m4s_file); + } else { + m4s_file = srs_path_build_timestamp(m4s_file); + } + if (true) { + std::stringstream ss; + ss << current_->sequence_no; + m4s_file = srs_string_replace(m4s_file, "[seq]", ss.str()); + } + + std::string m4s_path = hls_path_ + "/" + m4s_file; + current_->set_path(m4s_path); + + // the ts url, relative or absolute url. + // TODO: FIXME: Use url and path manager. + std::string m4s_url = current_->fullpath(); + if (srs_string_starts_with(m4s_url, m3u8_dir_)) { + m4s_url = m4s_url.substr(m3u8_dir_.length()); + } + while (srs_string_starts_with(m4s_url, "/")) { + m4s_url = m4s_url.substr(1); + } + + current_->uri += hls_entry_prefix_; + if (!hls_entry_prefix_.empty() && !srs_string_ends_with(hls_entry_prefix_, "/")) { + current_->uri += "/"; + + // add the http dir to uri. + string http_dir = srs_path_dirname(m3u8_url_); + if (!http_dir.empty()) { + current_->uri += http_dir + "/"; + } + } + current_->uri += m4s_url; + + current_->initialize(basetime, video_track_id_, audio_track_id_, sequence_no_, m4s_path); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::on_sequence_header() +{ + return srs_success; +} + +bool SrsHlsFmp4Muxer::is_segment_overflow() +{ + srs_assert(current_); + + // to prevent very small segment. + if (current_->duration() < 2 * SRS_HLS_SEGMENT_MIN_DURATION) { + return false; + } + + // Use N% deviation, to smoother. + srs_utime_t deviation = hls_ts_floor_ ? SRS_HLS_FLOOR_REAP_PERCENT * deviation_ts_ * hls_fragment_ : 0; + + // Keep in mind that we use max_td for the base duration, not the hls_fragment. To calculate + // max_td, multiply hls_fragment by hls_td_ratio. + return current_->duration() >= max_td_ + deviation; +} + +bool SrsHlsFmp4Muxer::wait_keyframe() +{ + return hls_wait_keyframe_; +} + +bool SrsHlsFmp4Muxer::is_segment_absolutely_overflow() +{ + srs_assert(current_); + + // to prevent very small segment. + if (current_->duration() < 2 * SRS_HLS_SEGMENT_MIN_DURATION) { + return false; + } + + // use N% deviation, to smoother. + srs_utime_t deviation = hls_ts_floor_? SRS_HLS_FLOOR_REAP_PERCENT * deviation_ts_ * hls_fragment_ : 0; + return current_->duration() >= hls_aof_ratio_ * hls_fragment_ + deviation; +} + +void SrsHlsFmp4Muxer::update_duration(uint64_t dts) +{ + current_->append(dts / 90); +} + +srs_error_t SrsHlsFmp4Muxer::segment_close() +{ + srs_error_t err = do_segment_close(); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::do_segment_close() +{ + srs_error_t err = srs_success; + + if (!current_) { + srs_warn("ignore the segment close, for segment is not open."); + return err; + } + + if ((err = current_->reap(video_dts_)) != srs_success) { + return srs_error_wrap(err, "reap segment"); + } + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHls(_srs_context->get_id(), req_, current_->fullpath(), + current_->uri, m3u8_, m3u8_url_, current_->sequence_no, current_->duration()))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + // use async to call the http hooks, for it will cause thread switch. + if ((err = async_->execute(new SrsDvrAsyncCallOnHlsNotify(_srs_context->get_id(), req_, current_->uri))) != srs_success) { + return srs_error_wrap(err, "segment close"); + } + + segments_->append(current_); + current_ = NULL; + + // shrink the segments. + segments_->shrink(hls_window_); + + // refresh the m3u8, donot contains the removed ts + if ((err = refresh_m3u8()) != srs_success) { + return srs_error_wrap(err, "refresh m3u8"); + } + + // remove the ts file. + segments_->clear_expired(hls_cleanup_); + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::write_hls_key() +{ + srs_error_t err = srs_success; + + if (hls_keys_ && current_->sequence_no % hls_fragments_per_key_ == 0) { + if (RAND_bytes(key_, 16) < 0) { + return srs_error_wrap(err, "rand key failed."); + } + if (RAND_bytes(kid_, 16) < 0) { + return srs_error_wrap(err, "rand kid failed."); + } + if (RAND_bytes(iv_, 16) < 0) { + return srs_error_wrap(err, "rand iv failed."); + } + + string key_file = srs_path_build_stream(hls_key_file_, req_->vhost, req_->app, req_->stream); + key_file = srs_string_replace(key_file, "[seq]", srs_int2str(current_->sequence_no)); + string key_url = hls_key_file_path_ + "/" + key_file; + + SrsFileWriter fw; + if ((err = fw.open(key_url)) != srs_success) { + return srs_error_wrap(err, "open file %s", key_url.c_str()); + } + + err = fw.write(key_, 16, NULL); + fw.close(); + + if (err != srs_success) { + return srs_error_wrap(err, "write key"); + } + } + + if (hls_keys_) { + current_->config_cipher(key_, iv_); + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::refresh_m3u8() +{ + srs_error_t err = srs_success; + + // no segments, also no m3u8, return. + if (segments_->empty()) { + return err; + } + + std::string temp_m3u8 = m3u8_ + ".temp"; + if ((err = _refresh_m3u8(temp_m3u8)) == srs_success) { + if (rename(temp_m3u8.c_str(), m3u8_.c_str()) < 0) { + err = srs_error_new(ERROR_HLS_WRITE_FAILED, "hls: rename m3u8 file failed. %s => %s", temp_m3u8.c_str(), m3u8_.c_str()); + } + } + + // remove the temp file. + if (srs_path_exists(temp_m3u8)) { + if (unlink(temp_m3u8.c_str()) < 0) { + srs_warn("ignore remove m3u8 failed, %s", temp_m3u8.c_str()); + } + } + + return err; +} + +srs_error_t SrsHlsFmp4Muxer::_refresh_m3u8(std::string m3u8_file) +{ + srs_error_t err = srs_success; + + // no segments, return. + if (segments_->empty()) { + return err; + } + + SrsFileWriter writer; + if ((err = writer.open(m3u8_file)) != srs_success) { + return srs_error_wrap(err, "hls: open m3u8 file %s", m3u8_file.c_str()); + } + + // #EXTM3U\n + // #EXT-X-VERSION:3\n + std::stringstream ss; + ss << "#EXTM3U" << SRS_CONSTS_LF; + // TODO: for fmp4 set #EXT-X-VERSION:7, need support tag #EXT-X-MAP:URI="init.mp4", which + // at least version:5 + // DOC: https://developer.apple.com/documentation/http-live-streaming/about-the-ext-x-version-tag + ss << "#EXT-X-VERSION:7" << SRS_CONSTS_LF; + + // #EXT-X-MEDIA-SEQUENCE:4294967295\n + SrsHlsM4sSegment* first = dynamic_cast(segments_->first()); + if (first == NULL) { + return srs_error_new(ERROR_HLS_WRITE_FAILED, "segments cast"); + } + + ss << "#EXT-X-MEDIA-SEQUENCE:" << first->sequence_no << SRS_CONSTS_LF; + + // #EXT-X-TARGETDURATION:4294967295\n + /** + * @see hls-m3u8-draft-pantos-http-live-streaming-12.pdf, page 25 + * The Media Playlist file MUST contain an EXT-X-TARGETDURATION tag. + * Its value MUST be equal to or greater than the EXTINF duration of any + * media segment that appears or will appear in the Playlist file, + * rounded to the nearest integer. Its value MUST NOT change. A + * typical target duration is 10 seconds. + */ + srs_utime_t max_duration = segments_->max_duration(); + int target_duration = (int)ceil(srsu2msi(srs_max(max_duration, max_td_)) / 1000.0); + + ss << "#EXT-X-TARGETDURATION:" << target_duration << SRS_CONSTS_LF; + + // TODO: add #EXT-X-MAP:URI="init.mp4" for fmp4 + ss << "#EXT-X-MAP:URI=\"" << init_mp4_uri_ << "\"" << SRS_CONSTS_LF; + + // write all segments + for (int i = 0; i < segments_->size(); i++) { + SrsHlsM4sSegment* segment = dynamic_cast(segments_->at(i)); + + if (segment->is_sequence_header()) { + // #EXT-X-DISCONTINUITY\n + ss << "#EXT-X-DISCONTINUITY" << SRS_CONSTS_LF; + } + +#if 1 + if(hls_keys_ && ((segment->sequence_no % hls_fragments_per_key_) == 0)) { + char hexiv[33]; + srs_data_to_hex(hexiv, segment->iv, 16); + hexiv[32] = '\0'; + + string key_file = srs_path_build_stream(hls_key_file_, req_->vhost, req_->app, req_->stream); + key_file = srs_string_replace(key_file, "[seq]", srs_int2str(segment->sequence_no)); + + string key_path = key_file; + //if key_url is not set,only use the file name + if (!hls_key_url_.empty()) { + key_path = hls_key_url_ + key_file; + } + + ss << "#EXT-X-KEY:METHOD=SAMPLE-AES,URI=" << "\"" << key_path << "\",IV=0x" << hexiv << SRS_CONSTS_LF; + } +#endif + + // "#EXTINF:4294967295.208,\n" + ss.precision(3); + ss.setf(std::ios::fixed, std::ios::floatfield); + ss << "#EXTINF:" << srsu2msi(segment->duration()) / 1000.0 << ", no desc" << SRS_CONSTS_LF; + + // {file name}\n + // TODO get segment name in relative path. + std::string seg_uri = segment->fullpath(); + if (true) { + std::stringstream stemp; + stemp << srsu2msi(segment->duration()); + seg_uri = srs_string_replace(seg_uri, "[duration]", stemp.str()); + } + //ss << segment->uri << SRS_CONSTS_LF; + ss << srs_path_basename(seg_uri) << SRS_CONSTS_LF; + } + + // write m3u8 to writer. + std::string m3u8 = ss.str(); + if ((err = writer.write((char*)m3u8.c_str(), (int)m3u8.length(), NULL)) != srs_success) { + return srs_error_wrap(err, "hls: write m3u8"); + } + + return err; +} + SrsHlsMuxer::SrsHlsMuxer() { req = NULL; @@ -800,6 +1670,9 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) // #EXT-X-VERSION:3\n std::stringstream ss; ss << "#EXTM3U" << SRS_CONSTS_LF; + // TODO: for fmp4 set #EXT-X-VERSION:7, need support tag #EXT-X-MAP:URI="init.mp4", which + // at least version:5 + // DOC: https://developer.apple.com/documentation/http-live-streaming/about-the-ext-x-version-tag ss << "#EXT-X-VERSION:3" << SRS_CONSTS_LF; // #EXT-X-MEDIA-SEQUENCE:4294967295\n @@ -823,6 +1696,8 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) int target_duration = (int)ceil(srsu2msi(srs_max(max_duration, max_td)) / 1000.0); ss << "#EXT-X-TARGETDURATION:" << target_duration << SRS_CONSTS_LF; + + // TODO: add #EXT-X-MAP:URI="init.mp4" for fmp4 // write all segments for (int i = 0; i < segments->size(); i++) { @@ -875,10 +1750,22 @@ srs_error_t SrsHlsMuxer::_refresh_m3u8(string m3u8_file) return err; } +ISrsHlsController::ISrsHlsController() +{ +} + +ISrsHlsController::~ISrsHlsController() +{ +} + SrsHlsController::SrsHlsController() { tsmc = new SrsTsMessageCache(); muxer = new SrsHlsMuxer(); + + hls_dts_directly = false; + previous_audio_dts = 0; + aac_samples = 0; } SrsHlsController::~SrsHlsController() @@ -972,7 +1859,9 @@ srs_error_t SrsHlsController::on_publish(SrsRequest* req) } // This config item is used in SrsHls, we just log its value here. - bool hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); + // If enabled, directly turn FLV timestamp to TS DTS. + // @remark It'll be reloaded automatically, because the origin hub will republish while reloading. + hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); srs_trace("hls: win=%dms, frag=%dms, prefix=%s, path=%s, m3u8=%s, ts=%s, tdr=%.2f, aof=%.2f, floor=%d, clean=%d, waitk=%d, dispose=%dms, dts_directly=%d", srsu2msi(hls_window), srsu2msi(hls_fragment), entry_prefix.c_str(), path.c_str(), m3u8_file.c_str(), ts_file.c_str(), @@ -1000,7 +1889,7 @@ srs_error_t SrsHlsController::on_unpublish() return err; } -srs_error_t SrsHlsController::on_sequence_header() +srs_error_t SrsHlsController::on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) { // TODO: support discontinuity for the same stream // currently we reap and insert discontinity when encoder republish, @@ -1011,10 +1900,50 @@ srs_error_t SrsHlsController::on_sequence_header() return muxer->on_sequence_header(); } -srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) +srs_error_t SrsHlsController::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) { srs_error_t err = srs_success; + SrsAudioFrame* frame = format->audio; + + // Reset the aac samples counter when DTS jitter. + if (previous_audio_dts > shared_audio->timestamp) { + previous_audio_dts = shared_audio->timestamp; + aac_samples = 0; + } + // The diff duration in ms between two FLV audio packets. + int diff = ::abs((int)(shared_audio->timestamp - previous_audio_dts)); + previous_audio_dts = shared_audio->timestamp; + + // Guess the number of samples for each AAC frame. + // If samples is 1024, the sample-rate is 8000HZ, the diff should be 1024/8000s=128ms. + // If samples is 1024, the sample-rate is 44100HZ, the diff should be 1024/44100s=23ms. + // If samples is 2048, the sample-rate is 44100HZ, the diff should be 2048/44100s=46ms. + int nb_samples_per_frame = 0; + int guessNumberOfSamples = diff * srs_flv_srates[format->acodec->sound_rate] / 1000; + if (guessNumberOfSamples > 0) { + if (guessNumberOfSamples < 960) { + nb_samples_per_frame = 960; + } else if (guessNumberOfSamples < 1536) { + nb_samples_per_frame = 1024; + } else if (guessNumberOfSamples < 3072) { + nb_samples_per_frame = 2048; + } else { + nb_samples_per_frame = 4096; + } + } + + // Recalc the DTS by the samples of AAC. + aac_samples += nb_samples_per_frame; + int64_t dts = 90000 * aac_samples / srs_flv_srates[format->acodec->sound_rate]; + + // If directly turn FLV timestamp, overwrite the guessed DTS. + // @doc https://github.com/ossrs/srs/issues/1506#issuecomment-562063095 + if (hls_dts_directly) { + dts = shared_audio->timestamp * 90; + } + + // Refresh the codec ASAP. if (muxer->latest_acodec() != frame->acodec()->id) { srs_trace("HLS: Switch audio codec %d(%s) to %d(%s)", muxer->latest_acodec(), srs_audio_codec_id2str(muxer->latest_acodec()).c_str(), @@ -1023,7 +1952,7 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) } // write audio to cache. - if ((err = tsmc->cache_audio(frame, pts)) != srs_success) { + if ((err = tsmc->cache_audio(frame, dts)) != srs_success) { return srs_error_wrap(err, "hls: cache audio"); } @@ -1046,7 +1975,7 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) // for pure audio, aggregate some frame to one. // TODO: FIXME: Check whether it's necessary. if (muxer->pure_audio() && tsmc->audio) { - if (pts - tsmc->audio->start_pts < SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE) { + if (dts - tsmc->audio->start_pts < SRS_CONSTS_HLS_PURE_AUDIO_AGGREGATE) { return err; } } @@ -1062,9 +1991,11 @@ srs_error_t SrsHlsController::write_audio(SrsAudioFrame* frame, int64_t pts) return err; } -srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts) +srs_error_t SrsHlsController::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) { srs_error_t err = srs_success; + SrsVideoFrame* frame = format->video; + int64_t dts = shared_video->timestamp * 90; // Refresh the codec ASAP. if (muxer->latest_vcodec() != frame->vcodec()->id) { @@ -1142,6 +2073,171 @@ srs_error_t SrsHlsController::reap_segment() return err; } +SrsHlsMp4Controller::SrsHlsMp4Controller() +{ + has_video_sh_ = false; + has_audio_sh_ = false; + + video_track_id_ = 1; + audio_track_id_ = 2; + + audio_dts_ = 0; + video_dts_ = 0; + + req_ = NULL; + muxer_ = new SrsHlsFmp4Muxer(); +} + +SrsHlsMp4Controller::~SrsHlsMp4Controller() +{ + srs_freep(muxer_); +} + +srs_error_t SrsHlsMp4Controller::initialize() +{ + srs_error_t err = srs_success; + if ((err = muxer_->initialize(video_track_id_, audio_track_id_)) != srs_success) { + return srs_error_wrap(err, "initialize SrsHlsFmp4Muxer"); + } + + return err; +} + +void SrsHlsMp4Controller::dispose() +{ + muxer_->dispose(); +} + +srs_error_t SrsHlsMp4Controller::on_publish(SrsRequest* req) +{ + srs_error_t err = srs_success; + + req_ = req; + std::string vhost = req->vhost; + std::string stream = req->stream; + std::string app = req->app; + + // get the hls m3u8 ts list entry prefix config + std::string entry_prefix = _srs_config->get_hls_entry_prefix(vhost); + // get the hls path config + std::string path = _srs_config->get_hls_path(vhost); + std::string m3u8_file = _srs_config->get_hls_m3u8_file(vhost); + std::string ts_file = _srs_config->get_hls_ts_file(vhost); + + if ((err = muxer_->on_publish(req)) != srs_success) { + return srs_error_wrap(err, "muxer publish"); + } + + if ((err = muxer_->update_config(req)) != srs_success ) { + return srs_error_wrap(err, "hls: update config"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::on_unpublish() +{ + srs_error_t err = srs_success; + req_ = NULL; + + if ((err = muxer_->segment_close()) != srs_success) { + return srs_error_wrap(err, "hls: segment close"); + } + + if ((err = muxer_->on_unpublish()) != srs_success) { + return srs_error_wrap(err, "muxer unpublish"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) +{ + srs_error_t err = srs_success; + + // Ignore audio sequence header + if (format->is_aac_sequence_header() || format->is_mp3_sequence_header()) { + return err; + } + + audio_dts_ = shared_audio->timestamp; + + if ((err = muxer_->write_audio(shared_audio, format)) != srs_success) { + return srs_error_wrap(err, "write audio"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) +{ + srs_error_t err = srs_success; + SrsVideoFrame* frame = format->video; + + // Refresh the codec ASAP. + if (muxer_->latest_vcodec() != frame->vcodec()->id) { + srs_trace("HLS: Switch video codec %d(%s) to %d(%s)", muxer_->latest_acodec(), srs_video_codec_id2str(muxer_->latest_vcodec()).c_str(), + frame->vcodec()->id, srs_video_codec_id2str(frame->vcodec()->id).c_str()); + muxer_->set_latest_vcodec(frame->vcodec()->id); + } + + video_dts_ = shared_video->timestamp; + + if ((err = muxer_->write_video(shared_video, format)) != srs_success) { + return srs_error_wrap(err, "write video"); + } + + return err; +} + +srs_error_t SrsHlsMp4Controller::on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) +{ + srs_error_t err = srs_success; + + if (req_ == NULL) { + return srs_error_new(ERROR_HLS_NO_STREAM, "no req yet"); + } + + if (msg->is_video()) { + has_video_sh_ = true; + } + + if (msg->is_audio()) { + if (format->acodec->aac_extra_data.size() == 0) { + srs_trace("the audio codec's aac extra data is empty"); + return err; + } + + has_audio_sh_ = true; + } + + if ((err = muxer_->write_init_mp4(format, has_video_sh_, has_audio_sh_)) != srs_success) { + return srs_error_wrap(err, "write init mp4"); + } + + return err; +} + +int SrsHlsMp4Controller::sequence_no() +{ + return muxer_->sequence_no(); +} + +std::string SrsHlsMp4Controller::ts_url() +{ + return muxer_->m4s_url(); +} + +srs_utime_t SrsHlsMp4Controller::duration() +{ + return muxer_->duration(); +} + +int SrsHlsMp4Controller::deviation() +{ + return muxer_->deviation(); +} + SrsHls::SrsHls() { req = NULL; @@ -1152,13 +2248,10 @@ SrsHls::SrsHls() unpublishing_ = false; async_reload_ = reloading_ = false; last_update_time = 0; - hls_dts_directly = false; - - previous_audio_dts = 0; - aac_samples = 0; jitter = new SrsRtmpJitter(); - controller = new SrsHlsController(); + // TODO: replace NULL by a dummy ISrsHlsController + controller = NULL; pprint = SrsPithyPrint::create_hls(); } @@ -1292,6 +2385,16 @@ srs_error_t SrsHls::initialize(SrsOriginHub* h, SrsRequest* r) hub = h; req = r; + + bool is_fmp4_enabled = _srs_config->get_hls_use_fmp4(r->vhost); + + if (!controller) { + if (is_fmp4_enabled) { + controller = new SrsHlsMp4Controller(); + } else { + controller = new SrsHlsController(); + } + } if ((err = controller->initialize()) != srs_success) { return srs_error_wrap(err, "controller initialize"); @@ -1319,10 +2422,6 @@ srs_error_t SrsHls::on_publish() if ((err = controller->on_publish(req)) != srs_success) { return srs_error_wrap(err, "hls: on publish"); } - - // If enabled, directly turn FLV timestamp to TS DTS. - // @remark It'll be reloaded automatically, because the origin hub will republish while reloading. - hls_dts_directly = _srs_config->get_vhost_hls_dts_directly(req->vhost); // if enabled, open the muxer. enabled = true; @@ -1367,6 +2466,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma // Ignore if no format->acodec, it means the codec is not parsed, or unknown codec. // @issue https://github.com/ossrs/srs/issues/1506#issuecomment-562079474 + // TODO: format->acodec is always not-nil, remove this check. if (!format->acodec) { return err; } @@ -1384,8 +2484,9 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma // ignore sequence header srs_assert(format->audio); - if (acodec == SrsAudioCodecIdAAC && format->audio->aac_packet_type == SrsAudioAacFrameTraitSequenceHeader) { - return controller->on_sequence_header(); + // TODO: verify mp3 play by HLS. + if (format->is_aac_sequence_header() || format->is_mp3_sequence_header()) { + return controller->on_sequence_header(audio.get(), format); } // TODO: FIXME: config the jitter of HLS. @@ -1393,45 +2494,7 @@ srs_error_t SrsHls::on_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* forma return srs_error_wrap(err, "hls: jitter"); } - // Reset the aac samples counter when DTS jitter. - if (previous_audio_dts > audio->timestamp) { - previous_audio_dts = audio->timestamp; - aac_samples = 0; - } - - // The diff duration in ms between two FLV audio packets. - int diff = ::abs((int)(audio->timestamp - previous_audio_dts)); - previous_audio_dts = audio->timestamp; - - // Guess the number of samples for each AAC frame. - // If samples is 1024, the sample-rate is 8000HZ, the diff should be 1024/8000s=128ms. - // If samples is 1024, the sample-rate is 44100HZ, the diff should be 1024/44100s=23ms. - // If samples is 2048, the sample-rate is 44100HZ, the diff should be 2048/44100s=46ms. - int nb_samples_per_frame = 0; - int guessNumberOfSamples = diff * srs_flv_srates[format->acodec->sound_rate] / 1000; - if (guessNumberOfSamples > 0) { - if (guessNumberOfSamples < 960) { - nb_samples_per_frame = 960; - } else if (guessNumberOfSamples < 1536) { - nb_samples_per_frame = 1024; - } else if (guessNumberOfSamples < 3072) { - nb_samples_per_frame = 2048; - } else { - nb_samples_per_frame = 4096; - } - } - - // Recalc the DTS by the samples of AAC. - aac_samples += nb_samples_per_frame; - int64_t dts = 90000 * aac_samples / srs_flv_srates[format->acodec->sound_rate]; - - // If directly turn FLV timestamp, overwrite the guessed DTS. - // @doc https://github.com/ossrs/srs/issues/1506#issuecomment-562063095 - if (hls_dts_directly) { - dts = audio->timestamp * 90; - } - - if ((err = controller->write_audio(format->audio, dts)) != srs_success) { + if ((err = controller->write_audio(audio.get(), format)) != srs_success) { return srs_error_wrap(err, "hls: write audio"); } @@ -1469,9 +2532,11 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma return err; } - // ignore sequence header - if (format->video->avc_packet_type == SrsVideoAvcFrameTraitSequenceHeader) { - return controller->on_sequence_header(); + // ignore sequence header avc and hevc + // is avc|hevc|av1 sequence header check, but av1 packet already ignored above. so it's ok to use + // below method. + if (format->is_avc_sequence_header()) { + return controller->on_sequence_header(video.get(), format); } // TODO: FIXME: config the jitter of HLS. @@ -1479,8 +2544,7 @@ srs_error_t SrsHls::on_video(SrsSharedPtrMessage* shared_video, SrsFormat* forma return srs_error_wrap(err, "hls: jitter"); } - int64_t dts = video->timestamp * 90; - if ((err = controller->write_video(format->video, dts)) != srs_success) { + if ((err = controller->write_video(video.get(), format)) != srs_success) { return srs_error_wrap(err, "hls: write video"); } diff --git a/trunk/src/app/srs_app_hls.hpp b/trunk/src/app/srs_app_hls.hpp index 6d815a63c..4f7f98baa 100644 --- a/trunk/src/app/srs_app_hls.hpp +++ b/trunk/src/app/srs_app_hls.hpp @@ -16,6 +16,7 @@ #include #include #include +#include class SrsFormat; class SrsSharedPtrMessage; @@ -32,11 +33,13 @@ class SrsTsAacJitter; class SrsTsMessageCache; class SrsHlsSegment; class SrsTsContext; +class SrsFmp4SegmentEncoder; // The wrapper of m3u8 segment from specification: // // 3.3.2. EXTINF // The EXTINF tag specifies the duration of a media segment. +// TODO: refactor this to support fmp4 segment. class SrsHlsSegment : public SrsFragment { public: @@ -56,11 +59,60 @@ public: SrsHlsSegment(SrsTsContext* c, SrsAudioCodecId ac, SrsVideoCodecId vc, SrsFileWriter* w); virtual ~SrsHlsSegment(); public: - void config_cipher(unsigned char* key,unsigned char* iv); + void config_cipher(unsigned char* key, unsigned char* iv); // replace the placeholder virtual srs_error_t rename(); }; +class SrsInitMp4Segment : public SrsFragment +{ +private: + SrsFileWriter* fw_; + SrsMp4M2tsInitEncoder init_; +private: + // Key ID for encryption + unsigned char kid_[16]; + // Constant IV for encryption + unsigned char const_iv_[16]; + // IV size (8 or 16 bytes) + uint8_t const_iv_size_; +public: + SrsInitMp4Segment(SrsFileWriter* fw); + virtual ~SrsInitMp4Segment(); +public: + virtual srs_error_t config_cipher(unsigned char* kid, unsigned char* const_iv, uint8_t const_iv_size); + // Write the init mp4 file, with the v_tid(video track id) and a_tid (audio track id). + virtual srs_error_t write(SrsFormat* format, int v_tid, int a_tid); + virtual srs_error_t write_video_only(SrsFormat* format, int v_tid); + virtual srs_error_t write_audio_only(SrsFormat* format, int a_tid); +private: + virtual srs_error_t init_encoder(); +}; + +// TODO: merge this code with SrsFragmentedMp4 in dash +class SrsHlsM4sSegment : public SrsFragment +{ +private: + SrsFileWriter* fw_; + SrsFmp4SegmentEncoder enc_; +public: + // m4s uri in m3u8. + std::string uri; + // sequence number in m3u8. + int sequence_no; + // IV for encryption, saved in m3u8 file. + unsigned char iv[16]; +public: + SrsHlsM4sSegment(SrsFileWriter* fw); + virtual ~SrsHlsM4sSegment(); +public: + virtual srs_error_t initialize(int64_t time, uint32_t v_tid, uint32_t a_tid, int sequence_number, std::string m4s_path); + virtual void config_cipher(unsigned char* key, unsigned char* iv); + virtual srs_error_t write(SrsSharedPtrMessage* shared_msg, SrsFormat* format); + // Finalizes segment + virtual srs_error_t reap(uint64_t dts); +}; + // The hls async call: on_hls class SrsDvrAsyncCallOnHls : public ISrsAsyncCallTask { @@ -103,6 +155,7 @@ public: // // That is, user must use HlsCache, which will control the methods of muxer, // and provides HLS mechenisms. +// TODO: Rename to SrsHlsTsMuxer, for TS file only. class SrsHlsMuxer { private: @@ -217,6 +270,157 @@ private: virtual srs_error_t _refresh_m3u8(std::string m3u8_file); }; +// Mux the HLS stream(m3u8 and m4s files). +// Generally, the m3u8 muxer only provides methods to open/close segments, +// to flush video/audio, without any mechenisms. +class SrsHlsFmp4Muxer +{ +private: + SrsRequest* req_; +private: + std::string hls_entry_prefix_; + std::string hls_path_; + std::string hls_m4s_file_; + bool hls_cleanup_; + bool hls_wait_keyframe_; + std::string m3u8_dir_; + double hls_aof_ratio_; + // TODO: FIXME: Use TBN 1000. + srs_utime_t hls_fragment_; + srs_utime_t hls_window_; + SrsAsyncCallWorker* async_; +private: + // Whether use floor algorithm for timestamp. + bool hls_ts_floor_; + // The deviation in piece to adjust the fragment to be more + // bigger or smaller. + int deviation_ts_; + // The previous reap floor timestamp, + // used to detect the dup or jmp or ts. + int64_t accept_floor_ts_; + int64_t previous_floor_ts_; + bool init_mp4_ready_; +private: + // Whether encrypted or not + // TODO: fmp4 encryption is not yet implemented. + // fmp4 support four kinds of protection scheme: 'cenc', 'cbc1', 'cens', 'cbcs'. + // @see: https://cdn.standards.iteh.ai/samples/84637/04ebded1a92a4c8ab9be6f419a3252ed/ISO-IEC-23001-7-2023.pdf + // But unfortunately the above link is just part of the spec, the full doc is not free. + // And Apple's doc said HLS support unencrypted and encrypted with 'cbcs'. + // @see: https://developer.apple.com/documentation/http-live-streaming/about-the-common-media-application-format-with-http-live-streaming-hls + // Another Apple doc said Encrypted fmp4 content MUST contain either a Sample Encryption Box('senc'), or both a Sample Auxiliary Information + // Sizes Box('saiz') and a Sample Auxiliary Information Offsets Box('saio'). + // @see: https://developer.apple.com/documentation/http-live-streaming/hls-authoring-specification-for-apple-devices + bool hls_keys_; + int hls_fragments_per_key_; + // The key file name + std::string hls_key_file_; + // The key file path + std::string hls_key_file_path_; + // The key file url + std::string hls_key_url_; + // The key and iv. + unsigned char key_[16]; + unsigned char kid_[16]; + unsigned char iv_[16]; + // The underlayer file writer. + SrsFileWriter* writer_; +private: + int sequence_no_; + srs_utime_t max_td_; + std::string m3u8_; + std::string m3u8_url_; + std::string init_mp4_uri_; // URI for init.mp4 in m3u8 playlist + int video_track_id_; + int audio_track_id_; + uint64_t video_dts_; +private: + // The available cached segments in m3u8. + SrsFragmentWindow* segments_; + // The current writing segment. + SrsHlsM4sSegment* current_; +private: + // Latest audio codec, parsed from stream. + SrsAudioCodecId latest_acodec_; + // Latest audio codec, parsed from stream. + SrsVideoCodecId latest_vcodec_; +public: + SrsHlsFmp4Muxer(); + virtual ~SrsHlsFmp4Muxer(); +public: + virtual void dispose(); +public: + virtual int sequence_no(); + virtual std::string m4s_url(); + virtual srs_utime_t duration(); + virtual int deviation(); +public: + SrsAudioCodecId latest_acodec(); + void set_latest_acodec(SrsAudioCodecId v); + SrsVideoCodecId latest_vcodec(); + void set_latest_vcodec(SrsVideoCodecId v); +public: + // Initialize the hls muxer. + virtual srs_error_t initialize(int v_tid, int a_tid); + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req); +public: + virtual srs_error_t write_init_mp4(SrsFormat* format, bool has_video, bool has_audio); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); +public: + virtual srs_error_t on_unpublish(); + // When publish, update the config for muxer. + virtual srs_error_t update_config(SrsRequest* r); + // Open a new segment(a new ts file) + virtual srs_error_t segment_open(srs_utime_t basetime); + virtual srs_error_t on_sequence_header(); + // Whether segment overflow, + // that is whether the current segment duration>=(the segment in config) + virtual bool is_segment_overflow(); + // Whether wait keyframe to reap the ts. + virtual bool wait_keyframe(); + // Whether segment absolutely overflow, for pure audio to reap segment, + // that is whether the current segment duration>=2*(the segment in config) + virtual bool is_segment_absolutely_overflow(); +public: + // When flushing video or audio, we update the duration. But, we should also update the + // duration before closing the segment. Keep in mind that it's fine to update the duration + // several times using the same dts timestamp. + void update_duration(uint64_t dts); + // Close segment(ts). + virtual srs_error_t segment_close(); +private: + virtual srs_error_t do_segment_close(); + virtual srs_error_t write_hls_key(); + virtual srs_error_t refresh_m3u8(); + virtual srs_error_t _refresh_m3u8(std::string m3u8_file); +}; + +// The base class for HLS controller +class ISrsHlsController +{ +public: + ISrsHlsController(); + virtual ~ISrsHlsController(); +public: + virtual srs_error_t initialize() = 0; + virtual void dispose() = 0; + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req) = 0; + virtual srs_error_t on_unpublish() = 0; +public: + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format) = 0; + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format) = 0; +public: + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* msg, SrsFormat* format) = 0; + virtual int sequence_no() = 0; + // TODO: maybe rename to segment_url? + virtual std::string ts_url() = 0; + virtual srs_utime_t duration() = 0; + virtual int deviation() = 0; +}; + // The hls stream cache, // use to cache hls stream and flush to hls muxer. // @@ -232,7 +436,8 @@ private: // when timestamp convert to flv tbn, it will loose precise, // so we must gather audio frame together, and recalc the timestamp @see SrsTsAacJitter, // we use a aac jitter to correct the audio pts. -class SrsHlsController +// TODO: Rename to SrsHlsTsController, for TS file only. +class SrsHlsController : public ISrsHlsController { private: // The HLS muxer to reap ts and m3u8. @@ -240,6 +445,14 @@ private: SrsHlsMuxer* muxer; // The TS cache SrsTsMessageCache* tsmc; + + // If the diff=dts-previous_audio_dts is about 23, + // that's the AAC samples is 1024, and we use the samples to calc the dts. + int64_t previous_audio_dts; + // The total aac samples. + uint64_t aac_samples; + // Whether directly turn FLV timestamp to TS DTS. + bool hls_dts_directly; public: SrsHlsController(); virtual ~SrsHlsController(); @@ -258,11 +471,11 @@ public: // must write a #EXT-X-DISCONTINUITY to m3u8. // @see: hls-m3u8-draft-pantos-http-live-streaming-12.txt // @see: 3.4.11. EXT-X-DISCONTINUITY - virtual srs_error_t on_sequence_header(); + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format); // write audio to cache, if need to flush, flush to muxer. - virtual srs_error_t write_audio(SrsAudioFrame* frame, int64_t pts); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); // write video to muxer. - virtual srs_error_t write_video(SrsVideoFrame* frame, int64_t dts); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); private: // Reopen the muxer for a new hls segment, // close current segment, open a new segment, @@ -271,12 +484,50 @@ private: virtual srs_error_t reap_segment(); }; -// Transmux RTMP stream to HLS(m3u8 and ts). +// HLS controller for fMP4 (.m4s) segments with init.mp4. +// Direct sample processing without caching, simpler than TS controller. +class SrsHlsMp4Controller : public ISrsHlsController +{ +private: + bool has_video_sh_; + bool has_audio_sh_; +private: + int video_track_id_; + int audio_track_id_; +private: + // Current audio dts. + uint64_t audio_dts_; + // Current video dts. + uint64_t video_dts_; +private: + SrsRequest* req_; +private: + SrsHlsFmp4Muxer* muxer_; +public: + SrsHlsMp4Controller(); + virtual ~SrsHlsMp4Controller(); +public: + virtual srs_error_t initialize(); + virtual void dispose(); + // When publish or unpublish stream. + virtual srs_error_t on_publish(SrsRequest* req); + virtual srs_error_t on_unpublish(); + virtual srs_error_t write_audio(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual srs_error_t write_video(SrsSharedPtrMessage* shared_video, SrsFormat* format); +public: + virtual srs_error_t on_sequence_header(SrsSharedPtrMessage* shared_audio, SrsFormat* format); + virtual int sequence_no(); + virtual std::string ts_url(); + virtual srs_utime_t duration(); + virtual int deviation(); +}; + +// Transmux RTMP stream to HLS(m3u8 and ts,fmp4). // TODO: FIXME: add utest for hls. class SrsHls { private: - SrsHlsController* controller; + ISrsHlsController* controller; private: SrsRequest* req; // Whether the HLS is enabled. @@ -290,14 +541,6 @@ private: bool reloading_; // To detect heartbeat and dispose it if configured. srs_utime_t last_update_time; -private: - // If the diff=dts-previous_audio_dts is about 23, - // that's the AAC samples is 1024, and we use the samples to calc the dts. - int64_t previous_audio_dts; - // The total aac samples. - uint64_t aac_samples; - // Whether directly turn FLV timestamp to TS DTS. - bool hls_dts_directly; private: SrsOriginHub* hub; SrsRtmpJitter* jitter; diff --git a/trunk/src/app/srs_app_http_static.cpp b/trunk/src/app/srs_app_http_static.cpp index bfa407471..55181857a 100644 --- a/trunk/src/app/srs_app_http_static.cpp +++ b/trunk/src/app/srs_app_http_static.cpp @@ -222,10 +222,11 @@ srs_error_t SrsHlsStream::serve_exists_session(ISrsHttpResponseWriter* w, ISrsHt } // Rebuild the m3u8 content, make .ts with hls_ctx. - size_t pos_ts = content.find(".ts"); static string QUERY_PREFIX = string(".ts?") + string(SRS_CONTEXT_IN_HLS) + string("="); - - if (pos_ts != string::npos) { + static string M4S_QUERY_PREFIX = string(".m4s?") + string(SRS_CONTEXT_IN_HLS) + string("="); + static string INIT_MP4_QUERY_PREFIX = string("init.mp4?") + string(SRS_CONTEXT_IN_HLS) + string("="); + + if (content.find(".ts") != string::npos) { string ctx = r->query_get(SRS_CONTEXT_IN_HLS); string query = QUERY_PREFIX + ctx; @@ -236,6 +237,28 @@ srs_error_t SrsHlsStream::serve_exists_session(ISrsHttpResponseWriter* w, ISrsHt } else { content = srs_string_replace(content, ".ts", query); } + } else if (content.find(".m4s") != string::npos) { + string ctx = r->query_get(SRS_CONTEXT_IN_HLS); + string query = M4S_QUERY_PREFIX + ctx; + + size_t pos_query = content.find(".m4s?"); + if (pos_query != string::npos) { + query += "&"; + content = srs_string_replace(content, ".m4s?", query); + } else { + content = srs_string_replace(content, ".m4s", query); + } + } else if (content.find("init.mp4") != string::npos) { + string ctx = r->query_get(SRS_CONTEXT_IN_HLS); + string query = INIT_MP4_QUERY_PREFIX + ctx; + + size_t pos_query = content.find("init.mp4?"); + if (pos_query != string::npos) { + query += "&"; + content = srs_string_replace(content, "init.mp4?", query); + } else { + content = srs_string_replace(content, "init.mp4", query); + } } // Response with rebuilt content. diff --git a/trunk/src/app/srs_app_http_static.hpp b/trunk/src/app/srs_app_http_static.hpp index 5d1e243a7..acf310717 100644 --- a/trunk/src/app/srs_app_http_static.hpp +++ b/trunk/src/app/srs_app_http_static.hpp @@ -74,6 +74,7 @@ protected: virtual srs_error_t serve_mp4_stream(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath, int64_t start, int64_t end); // Support HLS streaming with pseudo session id. virtual srs_error_t serve_m3u8_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); + // the ts file including: .ts .m4s init.mp4 virtual srs_error_t serve_ts_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); }; diff --git a/trunk/src/app/srs_app_server.hpp b/trunk/src/app/srs_app_server.hpp index 801990458..d56ec6ae4 100644 --- a/trunk/src/app/srs_app_server.hpp +++ b/trunk/src/app/srs_app_server.hpp @@ -130,8 +130,10 @@ private: SrsTcpListener* https_listener_; // WebRTC over TCP listener. Please note that there is always a UDP listener by RTC server. SrsTcpListener* webrtc_listener_; +#ifdef SRS_RTSP // RTSP listener, over TCP. SrsTcpListener* rtsp_listener_; +#endif // Stream Caster for push over HTTP-FLV. SrsHttpFlvListener* stream_caster_flv_listener_; // Stream Caster for push over MPEGTS-UDP diff --git a/trunk/src/core/srs_core_version7.hpp b/trunk/src/core/srs_core_version7.hpp index 1ee7bc906..8e3a9a539 100644 --- a/trunk/src/core/srs_core_version7.hpp +++ b/trunk/src/core/srs_core_version7.hpp @@ -9,6 +9,6 @@ #define VERSION_MAJOR 7 #define VERSION_MINOR 0 -#define VERSION_REVISION 50 +#define VERSION_REVISION 51 #endif \ No newline at end of file diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 638bacd60..f44fa4fe4 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -1378,6 +1378,7 @@ public: public: virtual bool is_aac_sequence_header(); virtual bool is_mp3_sequence_header(); + // TODO: is avc|hevc|av1 sequence header virtual bool is_avc_sequence_header(); private: // Demux the video packet in H.264 codec. diff --git a/trunk/src/kernel/srs_kernel_mp4.cpp b/trunk/src/kernel/srs_kernel_mp4.cpp index 9a4ad4773..79a718b1f 100644 --- a/trunk/src/kernel/srs_kernel_mp4.cpp +++ b/trunk/src/kernel/srs_kernel_mp4.cpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -759,15 +760,8 @@ void SrsMp4MovieFragmentBox::set_mfhd(SrsMp4MovieFragmentHeaderBox* v) boxes.push_back(v); } -SrsMp4TrackFragmentBox* SrsMp4MovieFragmentBox::traf() +void SrsMp4MovieFragmentBox::add_traf(SrsMp4TrackFragmentBox* v) { - SrsMp4Box* box = get(SrsMp4BoxTypeTRAF); - return dynamic_cast(box); -} - -void SrsMp4MovieFragmentBox::set_traf(SrsMp4TrackFragmentBox* v) -{ - remove(SrsMp4BoxTypeTRAF); boxes.push_back(v); } @@ -1647,15 +1641,8 @@ SrsMp4MovieExtendsBox::~SrsMp4MovieExtendsBox() { } -SrsMp4TrackExtendsBox* SrsMp4MovieExtendsBox::trex() +void SrsMp4MovieExtendsBox::add_trex(SrsMp4TrackExtendsBox* v) { - SrsMp4Box* box = get(SrsMp4BoxTypeTREX); - return dynamic_cast(box); -} - -void SrsMp4MovieExtendsBox::set_trex(SrsMp4TrackExtendsBox* v) -{ - remove(SrsMp4BoxTypeTREX); boxes.push_back(v); } @@ -4790,6 +4777,666 @@ stringstream& SrsMp4SegmentIndexBox::dumps_detail(stringstream& ss, SrsMp4DumpCo return ss; } +SrsMp4SampleAuxiliaryInfoSizeBox::SrsMp4SampleAuxiliaryInfoSizeBox() +{ + type = SrsMp4BoxTypeSAIZ; +} + +SrsMp4SampleAuxiliaryInfoSizeBox::~SrsMp4SampleAuxiliaryInfoSizeBox() +{ +} + +int SrsMp4SampleAuxiliaryInfoSizeBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header(); + + if (flags & 0x01) { + size += 8; // add sizeof(aux_info_type) + sizeof(aux_info_type_parameter); + } + + size += 1; // sizeof(default_sample_info_size); + size += 4; // sizeof(sample_count); + + if (default_sample_info_size == 0) { + size += sample_info_sizes.size(); + } + + return size; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoSizeBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + if (flags & 0x01) { + buf->write_4bytes(aux_info_type); + buf->write_4bytes(aux_info_type_parameter); + } + + buf->write_1bytes(default_sample_info_size); + + if (default_sample_info_size == 0) { + buf->write_4bytes(sample_info_sizes.size()); + vector::iterator it; + for (it = sample_info_sizes.begin(); it != sample_info_sizes.end(); ++it) + { + buf->write_1bytes(*it); + } + } else { + buf->write_4bytes(sample_count); + } + + return err; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoSizeBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + if (flags & 0x01) { + aux_info_type = buf->read_4bytes(); + aux_info_type_parameter = buf->read_4bytes(); + } + + default_sample_info_size = buf->read_1bytes(); + sample_count = buf->read_4bytes(); + + if (default_sample_info_size == 0) { + for (int i = 0; i < sample_count; i++) { + sample_info_sizes.push_back(buf->read_1bytes()); + } + } + + return err; +} + +std::stringstream& SrsMp4SampleAuxiliaryInfoSizeBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "default_sample_info_size=" << (int)default_sample_info_size << ", sample_count=" << sample_count; + return ss; +} + +SrsMp4SampleAuxiliaryInfoOffsetBox::SrsMp4SampleAuxiliaryInfoOffsetBox() +{ + type = SrsMp4BoxTypeSAIO; +} + +SrsMp4SampleAuxiliaryInfoOffsetBox::~SrsMp4SampleAuxiliaryInfoOffsetBox() +{ +} + +int SrsMp4SampleAuxiliaryInfoOffsetBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header(); + + if (flags & 0x01) { + size += 8; // sizeof(aux_info_type) + sizeof(aux_info_type_parameter); + } + + size += 4; // sizeof(entry_count); + if (version == 0) { + size += offsets.size() * 4; + } else { + size += offsets.size() * 8; + } + + return size; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoOffsetBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + if (flags & 0x01) { + buf->write_4bytes(aux_info_type); + buf->write_4bytes(aux_info_type_parameter); + } + + buf->write_4bytes(offsets.size()); + vector::iterator it; + for (it = offsets.begin(); it != offsets.end(); ++it) + { + if (version == 0) { + buf->write_4bytes(*it); + } else { + buf->write_8bytes(*it); + } + } + + return err; +} + +srs_error_t SrsMp4SampleAuxiliaryInfoOffsetBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + if (flags & 0x01) { + aux_info_type = buf->read_4bytes(); + aux_info_type_parameter = buf->read_4bytes(); + } + + uint32_t entry_count = buf->read_4bytes(); + for (int i = 0; i < entry_count; i++) + { + if (version == 0) { + offsets.push_back(buf->read_4bytes()); + } else { + offsets.push_back(buf->read_8bytes()); + } + + } + + return err; +} + +std::stringstream& SrsMp4SampleAuxiliaryInfoOffsetBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "entry_count=" << offsets.size(); + return ss; +} + +SrsMp4SubSampleEncryptionInfo::SrsMp4SubSampleEncryptionInfo() +{ + bytes_of_clear_data = 0; + bytes_of_protected_data = 0; +} + +SrsMp4SubSampleEncryptionInfo::~SrsMp4SubSampleEncryptionInfo() +{ +} + +uint64_t SrsMp4SubSampleEncryptionInfo::nb_bytes() +{ + // sizeof(bytes_of_clear_data) + sizeof(bytes_of_protected_data); + return 6; +} + +srs_error_t SrsMp4SubSampleEncryptionInfo::encode(SrsBuffer* buf) +{ + buf->write_2bytes(bytes_of_clear_data); + buf->write_4bytes(bytes_of_protected_data); + + return srs_success; +} + +srs_error_t SrsMp4SubSampleEncryptionInfo::decode(SrsBuffer* buf) +{ + bytes_of_clear_data = buf->read_2bytes(); + bytes_of_protected_data = buf->read_4bytes(); + + return srs_success; +} + +std::stringstream& SrsMp4SubSampleEncryptionInfo::dumps(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "bytes_of_clear_data=" << bytes_of_clear_data << ", bytes_of_protected_data=" << bytes_of_protected_data; + return ss; +} + +SrsMp4SampleEncryptionEntry::SrsMp4SampleEncryptionEntry(SrsMp4FullBox* senc, uint8_t per_sample_iv_size) +{ + senc_ = senc; + srs_assert(per_sample_iv_size == 0 || per_sample_iv_size == 8 || per_sample_iv_size == 16); + per_sample_iv_size_ = per_sample_iv_size; + iv_ = (uint8_t*) malloc(per_sample_iv_size); +} + +SrsMp4SampleEncryptionEntry::~SrsMp4SampleEncryptionEntry() +{ + free(iv_); + iv_ = NULL; +} + +srs_error_t SrsMp4SampleEncryptionEntry::set_iv(uint8_t* iv, uint8_t iv_size) +{ + srs_assert(iv_size == per_sample_iv_size_); + memcpy(iv_, iv, iv_size); + + return srs_success; +} + +uint64_t SrsMp4SampleEncryptionEntry::nb_bytes() +{ + uint64_t size = per_sample_iv_size_; + if (senc_->flags & SrsMp4CencSampleEncryptionUseSubSample) { + size += 2; // size of subsample_count + size += subsample_infos.size() * 6; + } + + return size; +} + +srs_error_t SrsMp4SampleEncryptionEntry::encode(SrsBuffer* buf) +{ + if (per_sample_iv_size_ != 0) { + buf->write_bytes((char*) iv_, per_sample_iv_size_); + } + + if (senc_->flags & SrsMp4CencSampleEncryptionUseSubSample) { + buf->write_2bytes(subsample_infos.size()); + + vector::iterator it; + for (it = subsample_infos.begin(); it != subsample_infos.end(); ++it) { + (*it).encode(buf); + } + } + + return srs_success; +} + +srs_error_t SrsMp4SampleEncryptionEntry::decode(SrsBuffer* buf) +{ + if (per_sample_iv_size_ > 0) { + buf->read_bytes((char*)iv_, per_sample_iv_size_); + } + + if (senc_->flags & SrsMp4CencSampleEncryptionUseSubSample) { + uint16_t subsample_count = buf->read_2bytes(); + for (uint16_t i = 0; i < subsample_count; i++) { + SrsMp4SubSampleEncryptionInfo info; + info.decode(buf); + subsample_infos.push_back(info); + } + } + return srs_success; +} + +std::stringstream& SrsMp4SampleEncryptionEntry::dumps(std::stringstream& ss, SrsMp4DumpContext dc) +{ + // TODO: dump what? + ss << "iv=" << iv_ << endl; + + vector::iterator it; + for (it = subsample_infos.begin(); it != subsample_infos.end(); ++it) { + (*it).dumps(ss, dc); + ss << endl; + } + + return ss; +} + +SrsMp4SampleEncryptionBox::SrsMp4SampleEncryptionBox(uint8_t per_sample_iv_size) +{ + version = 0; + flags = SrsMp4CencSampleEncryptionUseSubSample; + type = SrsMp4BoxTypeSENC; + srs_assert(per_sample_iv_size == 0 || per_sample_iv_size == 8 || per_sample_iv_size == 16); + per_sample_iv_size_ = per_sample_iv_size; +} + +SrsMp4SampleEncryptionBox::~SrsMp4SampleEncryptionBox() +{ + vector::iterator it; + for (it = entries.begin(); it != entries.end(); it++) + { + SrsMp4SampleEncryptionEntry* entry = *it; + srs_freep(entry); + } + entries.clear(); +} + +int SrsMp4SampleEncryptionBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header() + 4; + + vector::iterator it; + for (it = entries.begin(); it < entries.end(); it++) + { + size += (*it)->nb_bytes(); + } + + return size; +} + +srs_error_t SrsMp4SampleEncryptionBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_4bytes(entries.size()); + vector::iterator it; + for (it = entries.begin(); it != entries.end(); it++) + { + (*it)->encode(buf); + } + + return err; +} + +srs_error_t SrsMp4SampleEncryptionBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + vector::iterator it; + for (it = entries.begin(); it != entries.end(); it++) + { + SrsMp4SampleEncryptionEntry* entry = *it; + srs_freep(entry); + } + entries.clear(); + + int32_t size = buf->read_4bytes(); + for (int i = 0; i < size; i++) { + SrsMp4SampleEncryptionEntry *entry = new SrsMp4SampleEncryptionEntry(this, per_sample_iv_size_); + entry->decode(buf); + entries.push_back(entry); + } + + return err; +} + +std::stringstream& SrsMp4SampleEncryptionBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "sample_count=" << entries.size() << endl; + return ss; +} + +SrsMp4ProtectionSchemeInfoBox::SrsMp4ProtectionSchemeInfoBox() +{ + type = SrsMp4BoxTypeSINF; +} + +SrsMp4ProtectionSchemeInfoBox::~SrsMp4ProtectionSchemeInfoBox() +{ +} + +SrsMp4OriginalFormatBox* SrsMp4ProtectionSchemeInfoBox::frma() +{ + SrsMp4Box* box = get(SrsMp4BoxTypeFRMA); + return dynamic_cast(box); +} + +void SrsMp4ProtectionSchemeInfoBox::set_frma(SrsMp4OriginalFormatBox* v) +{ + remove(SrsMp4BoxTypeFRMA); + boxes.push_back(v); +} + +SrsMp4SchemeTypeBox* SrsMp4ProtectionSchemeInfoBox::schm() +{ + SrsMp4Box* box = get(SrsMp4BoxTypeSCHM); + return dynamic_cast(box); +} + +void SrsMp4ProtectionSchemeInfoBox::set_schm(SrsMp4SchemeTypeBox* v) +{ + remove(SrsMp4BoxTypeSCHM); + boxes.push_back(v); +} + +SrsMp4SchemeInfoBox* SrsMp4ProtectionSchemeInfoBox::schi() +{ + SrsMp4Box* box = get(SrsMp4BoxTypeSCHI); + return dynamic_cast(box); +} + +void SrsMp4ProtectionSchemeInfoBox::set_schi(SrsMp4SchemeInfoBox* v) +{ + remove(SrsMp4BoxTypeSCHI); + boxes.push_back(v); +} + + +SrsMp4OriginalFormatBox::SrsMp4OriginalFormatBox(uint32_t original_format) +{ + type = SrsMp4BoxTypeFRMA; + data_format_ = original_format; +} + +SrsMp4OriginalFormatBox::~SrsMp4OriginalFormatBox() +{ +} + +int SrsMp4OriginalFormatBox::nb_header() +{ + return SrsMp4Box::nb_header() + 4; +} + +srs_error_t SrsMp4OriginalFormatBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_4bytes(data_format_); + + return err; +} + +srs_error_t SrsMp4OriginalFormatBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4Box::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + + data_format_ = buf->read_4bytes(); + + return err; +} + +std::stringstream& SrsMp4OriginalFormatBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "original format=" << data_format_ << endl; + return ss; +} + +SrsMp4SchemeTypeBox::SrsMp4SchemeTypeBox() +{ + type = SrsMp4BoxTypeSCHM; + scheme_uri_size = 0; +} + +SrsMp4SchemeTypeBox::~SrsMp4SchemeTypeBox() +{ +} + +void SrsMp4SchemeTypeBox::set_scheme_uri(char* uri, uint32_t uri_size) +{ + srs_assert(uri_size < SCHM_SCHEME_URI_MAX_SIZE); + memcpy(scheme_uri, uri, uri_size); + scheme_uri_size = uri_size; + scheme_uri[uri_size] = '\0'; +} + +int SrsMp4SchemeTypeBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header() + 4 + 4; // sizeof(scheme_type) + sizeof(scheme_version) + + if (flags & 0x01) { + size += scheme_uri_size; + } + + return size; +} + +srs_error_t SrsMp4SchemeTypeBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_4bytes(scheme_type); + buf->write_4bytes(scheme_version); + + if (flags & 0x01) { + buf->write_bytes(scheme_uri, scheme_uri_size); + buf->write_1bytes(0); + } + + return err; +} + +srs_error_t SrsMp4SchemeTypeBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "decode header"); + } + scheme_type = buf->read_4bytes(); + scheme_version = buf->read_4bytes(); + + if (flags & 0x01) { + memset(scheme_uri, 0, SCHM_SCHEME_URI_MAX_SIZE); + int s = 0; + while (s < SCHM_SCHEME_URI_MAX_SIZE-1) { + char c = buf->read_1bytes(); + scheme_uri[s] = c; + s++; + if (c == '\0') { + break; + } + } + scheme_uri_size = s; + } + + return err; +} + +std::stringstream& SrsMp4SchemeTypeBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + ss << "scheme_type=" << scheme_type << ", scheme_version=" << scheme_version << endl; + if (flags & 0x01) { + ss << "scheme_uri=" << scheme_uri << endl; + } + + return ss; +} + +SrsMp4SchemeInfoBox::SrsMp4SchemeInfoBox() +{ + type = SrsMp4BoxTypeSCHI; +} + +SrsMp4SchemeInfoBox::~SrsMp4SchemeInfoBox() +{ +} + +SrsMp4TrackEncryptionBox::SrsMp4TrackEncryptionBox() +{ + type = SrsMp4BoxTypeTENC; +} + +SrsMp4TrackEncryptionBox::~SrsMp4TrackEncryptionBox() +{ +} + +void SrsMp4TrackEncryptionBox::set_default_constant_IV(uint8_t* iv, uint8_t iv_size) +{ + srs_assert(iv_size == 8 || iv_size == 16); + memcpy(default_constant_IV, iv, iv_size); + default_constant_IV_size = iv_size; +} + +int SrsMp4TrackEncryptionBox::nb_header() +{ + int size = SrsMp4FullBox::nb_header(); + size += 1; // sizeof(reserved) + size += 1; // sizeof(reserved_2) or sizeof(default_crypt_byte_block) + sizeof(default_skip_byte_block); + size += 1; // sizeof(default_isProtected); + size += 1; // sizeof(default_Per_Sample_IV_Size; + size += 16; // sizeof(default_KID); + if (default_is_protected == 1 && default_per_sample_IV_size == 0) { + size += 1 + default_constant_IV_size; // sizeof(default_constant_IV_size) + sizeof(default_constant_IV); + } + + return size; +} + +srs_error_t SrsMp4TrackEncryptionBox::encode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::encode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + + buf->write_1bytes(reserved); + if (version == 0) { + buf->write_1bytes(reserved_2); + } else { + buf->write_1bytes( (default_crypt_byte_block << 4) | (default_skip_byte_block & 0x0F)); + } + + buf->write_1bytes(default_is_protected); + buf->write_1bytes(default_per_sample_IV_size); + buf->write_bytes((char*)default_KID, 16); + if (default_is_protected == 1 && default_per_sample_IV_size == 0) { + buf->write_1bytes(default_constant_IV_size); + buf->write_bytes((char*)default_constant_IV, default_constant_IV_size); + } + + return err; +} + +srs_error_t SrsMp4TrackEncryptionBox::decode_header(SrsBuffer* buf) +{ + srs_error_t err = srs_success; + + if ((err = SrsMp4FullBox::decode_header(buf)) != srs_success) { + return srs_error_wrap(err, "encode header"); + } + reserved = buf->read_1bytes(); + if (version == 0) { + reserved_2 = buf->read_1bytes(); + } else { + uint8_t v = buf->read_1bytes(); + default_crypt_byte_block = v >> 4; + default_skip_byte_block = v & 0x0f; + } + + default_is_protected = buf->read_1bytes(); + default_per_sample_IV_size = buf->read_1bytes(); + buf->read_bytes((char*)default_KID, 16); + + if (default_is_protected == 1 && default_per_sample_IV_size == 0) { + default_constant_IV_size = buf->read_1bytes(); + srs_assert(default_constant_IV_size == 8 || default_constant_IV_size == 16); + buf->read_bytes((char*) default_constant_IV, default_constant_IV_size); + } + + return err; +} + +std::stringstream& SrsMp4TrackEncryptionBox::dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc) +{ + if (version != 0) { + ss << "default_crypt_byte_block=" << default_crypt_byte_block << ", default_skip_byte_block=" << default_skip_byte_block << endl; + } + ss << "default_isProtected=" << default_is_protected << ", default_per_sample_IV_size=" << default_per_sample_IV_size << endl; + + return ss; +} + SrsMp4Sample::SrsMp4Sample() { type = SrsFrameTypeForbidden; @@ -4989,13 +5636,11 @@ srs_error_t SrsMp4SampleManager::write(SrsMp4MovieBox* moov) return err; } -srs_error_t SrsMp4SampleManager::write(SrsMp4MovieFragmentBox* moof, uint64_t dts) +srs_error_t SrsMp4SampleManager::write(SrsMp4TrackFragmentBox* traf, uint64_t dts) { srs_error_t err = srs_success; - SrsMp4TrackFragmentBox* traf = moof->traf(); SrsMp4TrackFragmentRunBox* trun = traf->trun(); - trun->flags = SrsMp4TrunFlagsDataOffset | SrsMp4TrunFlagsSampleDuration | SrsMp4TrunFlagsSampleSize | SrsMp4TrunFlagsSampleFlag | SrsMp4TrunFlagsSampleCtsOffset; @@ -6199,6 +6844,10 @@ SrsMp4ObjectType SrsMp4Encoder::get_audio_object_type() SrsMp4M2tsInitEncoder::SrsMp4M2tsInitEncoder() { writer = NULL; + crypt_byte_block_ = 0; + skip_byte_block_ = 0; + iv_size_ = 0; + is_protected_ = false; } SrsMp4M2tsInitEncoder::~SrsMp4M2tsInitEncoder() @@ -6211,6 +6860,18 @@ srs_error_t SrsMp4M2tsInitEncoder::initialize(ISrsWriter* w) return srs_success; } +void SrsMp4M2tsInitEncoder::config_encryption(uint8_t crypt_byte_block, uint8_t skip_byte_block, unsigned char* kid, unsigned char* iv, uint8_t iv_size) +{ + srs_assert(crypt_byte_block + skip_byte_block == 10); + srs_assert(iv_size == 8 || iv_size == 16); + crypt_byte_block_ = crypt_byte_block; + skip_byte_block_ = skip_byte_block; + memcpy(kid_, kid, 16); + memcpy(iv_, iv, iv_size); + iv_size_ = iv_size; + is_protected_ = true; +} + srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) { srs_error_t err = srs_success; @@ -6302,6 +6963,10 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) avc1->set_avcC(avcC); avcC->avc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(avc1)) != srs_success)) { + return srs_error_wrap(err, "encrypt avc1 box"); + } } else { SrsMp4VisualSampleEntry* hev1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); stsd->append(hev1); @@ -6314,6 +6979,10 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) hev1->set_hvcC(hvcC); hvcC->hevc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(hev1)) != srs_success)) { + return srs_error_wrap(err, "encrypt hev1 box"); + } } SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); @@ -6333,7 +7002,7 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) moov->set_mvex(mvex); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - mvex->set_trex(trex); + mvex->add_trex(trex); trex->track_ID = tid; trex->default_sample_description_index = 1; @@ -6404,6 +7073,10 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) SrsMp4EsdsBox* esds = new SrsMp4EsdsBox(); mp4a->set_esds(esds); + + if (is_protected_ && ((err = config_sample_description_encryption(mp4a)) != srs_success)) { + return srs_error_wrap(err, "encrypt mp4a box"); + } SrsMp4ES_Descriptor* es = esds->es; es->ES_ID = 0x02; @@ -6434,7 +7107,7 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) moov->set_mvex(mvex); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - mvex->set_trex(trex); + mvex->add_trex(trex); trex->track_ID = tid; trex->default_sample_description_index = 1; @@ -6448,6 +7121,317 @@ srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, bool video, int tid) return err; } +srs_error_t SrsMp4M2tsInitEncoder::write(SrsFormat* format, int v_tid, int a_tid) +{ + srs_error_t err = srs_success; + + // Write ftyp box. + if (true) { + SrsUniquePtr ftyp(new SrsMp4FileTypeBox()); + + ftyp->major_brand = SrsMp4BoxBrandMP42; // SrsMp4BoxBrandISO5; + ftyp->minor_version = 512; + ftyp->set_compatible_brands(SrsMp4BoxBrandISO6, SrsMp4BoxBrandMP41); + + if ((err = srs_mp4_write_box(writer, ftyp.get())) != srs_success) { + return srs_error_wrap(err, "write ftyp"); + } + } + + // Write moov. + if (true) { + SrsUniquePtr moov(new SrsMp4MovieBox()); + + SrsMp4MovieHeaderBox* mvhd = new SrsMp4MovieHeaderBox(); + moov->set_mvhd(mvhd); + + mvhd->timescale = 1000; // Use tbn ms. + mvhd->duration_in_tbn = 0; + mvhd->next_track_ID = 4294967295; // 2^32 - 1 + + // write video track + if (format->vcodec) { + SrsMp4TrackBox* trak = new SrsMp4TrackBox(); + moov->add_trak(trak); + + SrsMp4TrackHeaderBox* tkhd = new SrsMp4TrackHeaderBox(); + trak->set_tkhd(tkhd); + + tkhd->track_ID = v_tid; + tkhd->duration = 0; + tkhd->width = (format->vcodec->width << 16); + tkhd->height = (format->vcodec->height << 16); + + SrsMp4MediaBox* mdia = new SrsMp4MediaBox(); + trak->set_mdia(mdia); + + SrsMp4MediaHeaderBox* mdhd = new SrsMp4MediaHeaderBox(); + mdia->set_mdhd(mdhd); + + mdhd->timescale = 1000; + mdhd->duration = 0; + mdhd->set_language0('u'); + mdhd->set_language1('n'); + mdhd->set_language2('d'); + + SrsMp4HandlerReferenceBox* hdlr = new SrsMp4HandlerReferenceBox(); + mdia->set_hdlr(hdlr); + + hdlr->handler_type = SrsMp4HandlerTypeVIDE; + hdlr->name = "VideoHandler"; + + SrsMp4MediaInformationBox* minf = new SrsMp4MediaInformationBox(); + mdia->set_minf(minf); + + SrsMp4VideoMeidaHeaderBox* vmhd = new SrsMp4VideoMeidaHeaderBox(); + minf->set_vmhd(vmhd); + + SrsMp4DataInformationBox* dinf = new SrsMp4DataInformationBox(); + minf->set_dinf(dinf); + + SrsMp4DataReferenceBox* dref = new SrsMp4DataReferenceBox(); + dinf->set_dref(dref); + + SrsMp4DataEntryBox* url = new SrsMp4DataEntryUrlBox(); + dref->append(url); + + SrsMp4SampleTableBox* stbl = new SrsMp4SampleTableBox(); + minf->set_stbl(stbl); + + SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); + stbl->set_stsd(stsd); + + if (format->vcodec->id == SrsVideoCodecIdAVC) { + SrsMp4VisualSampleEntry* avc1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeAVC1); + stsd->append(avc1); + + avc1->width = format->vcodec->width; + avc1->height = format->vcodec->height; + avc1->data_reference_index = 1; + + SrsMp4AvccBox* avcC = new SrsMp4AvccBox(); + avc1->set_avcC(avcC); + + avcC->avc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(avc1)) != srs_success)) { + return srs_error_wrap(err, "encrypt avc1 box"); + } + } else { + SrsMp4VisualSampleEntry* hev1 = new SrsMp4VisualSampleEntry(SrsMp4BoxTypeHEV1); + stsd->append(hev1); + + hev1->width = format->vcodec->width; + hev1->height = format->vcodec->height; + hev1->data_reference_index = 1; + + SrsMp4HvcCBox* hvcC = new SrsMp4HvcCBox(); + hev1->set_hvcC(hvcC); + + hvcC->hevc_config = format->vcodec->avc_extra_data; + + if (is_protected_ && ((err = config_sample_description_encryption(hev1)) != srs_success)) { + return srs_error_wrap(err, "encrypt hev1 box"); + } + } + + SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); + stbl->set_stts(stts); + + SrsMp4Sample2ChunkBox* stsc = new SrsMp4Sample2ChunkBox(); + stbl->set_stsc(stsc); + + SrsMp4SampleSizeBox* stsz = new SrsMp4SampleSizeBox(); + stbl->set_stsz(stsz); + + // TODO: FIXME: need to check using stco or co64? + SrsMp4ChunkOffsetBox* stco = new SrsMp4ChunkOffsetBox(); + stbl->set_stco(stco); + } + + // write audio track + if (format->acodec) { + SrsMp4TrackBox* trak = new SrsMp4TrackBox(); + moov->add_trak(trak); + + SrsMp4TrackHeaderBox* tkhd = new SrsMp4TrackHeaderBox(); + tkhd->volume = 0x0100; + trak->set_tkhd(tkhd); + + tkhd->track_ID = a_tid; + tkhd->duration = 0; + + SrsMp4MediaBox* mdia = new SrsMp4MediaBox(); + trak->set_mdia(mdia); + + SrsMp4MediaHeaderBox* mdhd = new SrsMp4MediaHeaderBox(); + mdia->set_mdhd(mdhd); + + mdhd->timescale = 1000; + mdhd->duration = 0; + mdhd->set_language0('u'); + mdhd->set_language1('n'); + mdhd->set_language2('d'); + + SrsMp4HandlerReferenceBox* hdlr = new SrsMp4HandlerReferenceBox(); + mdia->set_hdlr(hdlr); + + hdlr->handler_type = SrsMp4HandlerTypeSOUN; + hdlr->name = "SoundHandler"; + + SrsMp4MediaInformationBox* minf = new SrsMp4MediaInformationBox(); + mdia->set_minf(minf); + + SrsMp4SoundMeidaHeaderBox* smhd = new SrsMp4SoundMeidaHeaderBox(); + minf->set_smhd(smhd); + + SrsMp4DataInformationBox* dinf = new SrsMp4DataInformationBox(); + minf->set_dinf(dinf); + + SrsMp4DataReferenceBox* dref = new SrsMp4DataReferenceBox(); + dinf->set_dref(dref); + + SrsMp4DataEntryBox* url = new SrsMp4DataEntryUrlBox(); + dref->append(url); + + SrsMp4SampleTableBox* stbl = new SrsMp4SampleTableBox(); + minf->set_stbl(stbl); + + SrsMp4SampleDescriptionBox* stsd = new SrsMp4SampleDescriptionBox(); + stbl->set_stsd(stsd); + + SrsMp4AudioSampleEntry* mp4a = new SrsMp4AudioSampleEntry(); + mp4a->data_reference_index = 1; + mp4a->samplerate = uint32_t(srs_flv_srates[format->acodec->sound_rate]) << 16; + if (format->acodec->sound_size == SrsAudioSampleBits16bit) { + mp4a->samplesize = 16; + } else { + mp4a->samplesize = 8; + } + if (format->acodec->sound_type == SrsAudioChannelsStereo) { + mp4a->channelcount = 2; + } else { + mp4a->channelcount = 1; + } + stsd->append(mp4a); + + SrsMp4EsdsBox* esds = new SrsMp4EsdsBox(); + mp4a->set_esds(esds); + if (is_protected_ && ((err = config_sample_description_encryption(mp4a)) != srs_success)) { + return srs_error_wrap(err, "encrypt mp4a box."); + } + + SrsMp4ES_Descriptor* es = esds->es; + es->ES_ID = 0x02; + + SrsMp4DecoderConfigDescriptor& desc = es->decConfigDescr; + desc.objectTypeIndication = SrsMp4ObjectTypeAac; + desc.streamType = SrsMp4StreamTypeAudioStream; + srs_freep(desc.decSpecificInfo); + + SrsMp4DecoderSpecificInfo* asc = new SrsMp4DecoderSpecificInfo(); + desc.decSpecificInfo = asc; + asc->asc = format->acodec->aac_extra_data; + + SrsMp4DecodingTime2SampleBox* stts = new SrsMp4DecodingTime2SampleBox(); + stbl->set_stts(stts); + + SrsMp4Sample2ChunkBox* stsc = new SrsMp4Sample2ChunkBox(); + stbl->set_stsc(stsc); + + SrsMp4SampleSizeBox* stsz = new SrsMp4SampleSizeBox(); + stbl->set_stsz(stsz); + + // TODO: FIXME: need to check using stco or co64? + SrsMp4ChunkOffsetBox* stco = new SrsMp4ChunkOffsetBox(); + stbl->set_stco(stco); + } + + if (true) { + SrsMp4MovieExtendsBox* mvex = new SrsMp4MovieExtendsBox(); + moov->set_mvex(mvex); + + // video trex + if (format->vcodec) { + SrsMp4TrackExtendsBox* v_trex = new SrsMp4TrackExtendsBox(); + mvex->add_trex(v_trex); + + v_trex->track_ID = v_tid; + v_trex->default_sample_description_index = 1; + } + + // audio trex + if (format->acodec) { + SrsMp4TrackExtendsBox* a_trex = new SrsMp4TrackExtendsBox(); + mvex->add_trex(a_trex); + + a_trex->track_ID = a_tid; + a_trex->default_sample_description_index = 1; + } + } + + if ((err = srs_mp4_write_box(writer, moov.get())) != srs_success) { + return srs_error_wrap(err, "write moov"); + } + } + + return err; +} + +/** + * box->type = 'encv' or 'enca' + * |encv| + * | |sinf| + * | | |frma| + * | | |schm| + * | | |schi| + * | | | |tenc| + */ +srs_error_t SrsMp4M2tsInitEncoder::config_sample_description_encryption(SrsMp4SampleEntry* box) +{ + srs_error_t err = srs_success; + + bool is_video_sample = false; + SrsMp4BoxType original_type = box->type; + + if (original_type == SrsMp4BoxTypeAVC1 || original_type == SrsMp4BoxTypeHEV1) + { + box->type = SrsMp4BoxTypeENCV; + is_video_sample = true; + } else if (original_type == SrsMp4BoxTypeMP4A) { + box->type = SrsMp4BoxTypeENCA; + } else { + return srs_error_new(ERROR_MP4_BOX_ILLEGAL_TYPE, "unknown sample type 0x%x to encrypt", original_type); + } + + SrsMp4ProtectionSchemeInfoBox* sinf = new SrsMp4ProtectionSchemeInfoBox(); + box->append(sinf); + + SrsMp4OriginalFormatBox* frma = new SrsMp4OriginalFormatBox(original_type); + sinf->set_frma(frma); + + SrsMp4SchemeTypeBox* schm = new SrsMp4SchemeTypeBox(); + schm->scheme_type = SrsMp4CENSchemeCBCS; + schm->scheme_version = 0x00010000; + sinf->set_schm(schm); + + SrsMp4SchemeInfoBox* schi = new SrsMp4SchemeInfoBox(); + SrsMp4TrackEncryptionBox* tenc = new SrsMp4TrackEncryptionBox(); + tenc->version = 1; + tenc->default_crypt_byte_block = is_video_sample ? crypt_byte_block_ : 0 ; + tenc->default_skip_byte_block = is_video_sample ? skip_byte_block_ : 0; + tenc->default_is_protected = 1; + tenc->default_per_sample_IV_size = 0; + tenc->default_constant_IV_size = iv_size_; + memcpy(tenc->default_constant_IV, iv_, iv_size_); + memcpy(tenc->default_KID, kid_, 16); + + schi->append(tenc); + sinf->set_schi(schi); + + return err; +} + SrsMp4M2tsSegmentEncoder::SrsMp4M2tsSegmentEncoder() { writer = NULL; @@ -6574,7 +7558,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) mfhd->sequence_number = sequence_number; SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); - moof->set_traf(traf); + moof->add_traf(traf); SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); traf->set_tfhd(tfhd); @@ -6591,7 +7575,7 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); traf->set_trun(trun); - if ((err = samples->write(moof.get(), dts)) != srs_success) { + if ((err = samples->write(traf, dts)) != srs_success) { return srs_error_wrap(err, "write samples"); } @@ -6641,3 +7625,254 @@ srs_error_t SrsMp4M2tsSegmentEncoder::flush(uint64_t& dts) return err; } +SrsFmp4SegmentEncoder::SrsFmp4SegmentEncoder() +{ + writer_ = NULL; + sequence_number_ = 0; + decode_basetime_ = 0; + audio_track_id_ = 0; + video_track_id_ = 0; + nb_audios_ = 0; + nb_videos_ = 0; + styp_bytes_ = 0; + mdat_audio_bytes_ = 0; + mdat_video_bytes_ = 0; + audio_samples_ = new SrsMp4SampleManager(); + video_samples_ = new SrsMp4SampleManager(); + + memset(iv_,0,16); + key_ = (unsigned char*)new AES_KEY(); + do_sample_encryption_ = false; +} + +SrsFmp4SegmentEncoder::~SrsFmp4SegmentEncoder() +{ + srs_freep(audio_samples_); + srs_freep(video_samples_); + + AES_KEY* k = (AES_KEY*)key_; + srs_freep(k); +} + + +srs_error_t SrsFmp4SegmentEncoder::initialize(ISrsWriter* w, uint32_t sequence, srs_utime_t basetime, uint32_t v_tid, uint32_t a_tid) +{ + srs_error_t err = srs_success; + + writer_ = w; + sequence_number_ = sequence; + decode_basetime_ = basetime; + video_track_id_ = v_tid; + audio_track_id_ = a_tid; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::config_cipher(unsigned char* key, unsigned char* iv) +{ + srs_error_t err = srs_success; + + memcpy(this->iv_, iv, 16); + + AES_KEY* k = (AES_KEY*)this->key_; + if (AES_set_encrypt_key(key, 16 * 8, k)) { + return srs_error_new(ERROR_SYSTEM_FILE_WRITE, "set aes key failed"); + } + do_sample_encryption_ = true; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::write_sample(SrsMp4HandlerType ht, uint16_t ft, + uint32_t dts, uint32_t pts, uint8_t* sample, uint32_t nb_sample) +{ + srs_error_t err = srs_success; + + SrsMp4Sample* ps = new SrsMp4Sample(); + + if (ht == SrsMp4HandlerTypeVIDE) { + ps->type = SrsFrameTypeVideo; + ps->frame_type = (SrsVideoAvcFrameType)ft; + ps->index = nb_videos_++; + video_samples_->append(ps); + mdat_video_bytes_ += nb_sample; + } else if (ht == SrsMp4HandlerTypeSOUN) { + ps->type = SrsFrameTypeAudio; + ps->index = nb_audios_++; + audio_samples_->append(ps); + mdat_audio_bytes_ += nb_sample; + } else { + srs_freep(ps); + return err; + } + + ps->tbn = 1000; + ps->dts = dts; + ps->pts = pts; + + // We should copy the sample data, which is shared ptr from video/audio message. + // Furthermore, we do free the data when freeing the sample. + ps->data = new uint8_t[nb_sample]; + memcpy(ps->data, sample, nb_sample); + ps->nb_data = nb_sample; + + return err; +} + +srs_error_t SrsFmp4SegmentEncoder::flush(uint64_t dts) +{ + srs_error_t err = srs_success; + SrsMp4TrackFragmentRunBox* video_trun = NULL; + SrsMp4TrackFragmentRunBox* audio_trun = NULL; + + if (nb_videos_ == 0 && nb_audios_ == 0) { + return srs_error_new(ERROR_MP4_ILLEGAL_MDAT, "empty samples"); + } + // Create a mdat box. + // its payload will be writen by samples, + // and we will update its header(size) when flush. + SrsUniquePtr mdat(new SrsMp4MediaDataBox()); + + SrsUniquePtr moof(new SrsMp4MovieFragmentBox()); + + SrsMp4MovieFragmentHeaderBox* mfhd = new SrsMp4MovieFragmentHeaderBox(); + moof->set_mfhd(mfhd); + mfhd->sequence_number = sequence_number_; + + // write video traf + if (mdat_video_bytes_ > 0) { + // video traf + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + moof->add_traf(traf); + + SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); + traf->set_tfhd(tfhd); + + tfhd->track_id = video_track_id_; + tfhd->flags = SrsMp4TfhdFlagsDefaultBaseIsMoof; + + SrsMp4TrackFragmentDecodeTimeBox* tfdt = new SrsMp4TrackFragmentDecodeTimeBox(); + traf->set_tfdt(tfdt); + + tfdt->version = 1; + tfdt->base_media_decode_time = srsu2ms(decode_basetime_); + + SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); + traf->set_trun(trun); + video_trun = trun; + + if ((err = video_samples_->write(traf, dts)) != srs_success) { + return srs_error_wrap(err, "write samples"); + } + + // TODO: write senc, and optional saiz & saio + if (do_sample_encryption_) { + SrsMp4SampleEncryptionBox* senc = new SrsMp4SampleEncryptionBox(0); + // video_samples_; + vector::iterator it; + // write video sample data + for (it = video_samples_->samples.begin(); it != video_samples_->samples.end(); ++it) { + // SrsMp4Sample* sample = *it; + // TODO: parse hevc|avc, nalu slice header, and calculate + // sample->data; + // sample->nb_data; + } + + traf->append(senc); + } + } + + // write audio traf + if (mdat_audio_bytes_ > 0) { + // audio traf + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + moof->add_traf(traf); + + SrsMp4TrackFragmentHeaderBox* tfhd = new SrsMp4TrackFragmentHeaderBox(); + traf->set_tfhd(tfhd); + + tfhd->track_id = audio_track_id_; + tfhd->flags = SrsMp4TfhdFlagsDefaultBaseIsMoof; + + SrsMp4TrackFragmentDecodeTimeBox* tfdt = new SrsMp4TrackFragmentDecodeTimeBox(); + traf->set_tfdt(tfdt); + + tfdt->version = 1; + tfdt->base_media_decode_time = srsu2ms(decode_basetime_); + + SrsMp4TrackFragmentRunBox* trun = new SrsMp4TrackFragmentRunBox(); + traf->set_trun(trun); + audio_trun = trun; + + if ((err = audio_samples_->write(traf, dts)) != srs_success) { + return srs_error_wrap(err, "write samples"); + } + + // TODO: write senc, and optional saiz & saio + if (do_sample_encryption_) { + SrsMp4SampleEncryptionBox* senc = new SrsMp4SampleEncryptionBox(0); + // this->iv_; + traf->append(senc); + } + } + + // @remark Remember the data_offset of turn is size(moof)+header(mdat) + int moof_bytes = moof->nb_bytes(); + // rewrite video data_offset + if (video_trun != NULL) { + video_trun->data_offset = (int32_t)(moof_bytes + mdat->sz_header() + 0); + } + + if (audio_trun != NULL) { + audio_trun->data_offset = (int32_t)(moof_bytes + mdat->sz_header() + mdat_video_bytes_); + } + + // srs_trace("seq: %d, moof_bytes=%d, mdat->sz_header=%d", sequence_number_, moof->nb_bytes(), mdat->sz_header()); + // srs_trace("mdat_video_bytes_ = %d, mdat_audio_bytes_ = %d", mdat_video_bytes_, mdat_audio_bytes_); + + if ((err = srs_mp4_write_box(writer_, moof.get())) != srs_success) { + return srs_error_wrap(err, "write moof"); + } + + mdat->nb_data = mdat_video_bytes_ + mdat_audio_bytes_; + // Write mdat. + if (true) { + int nb_data = mdat->sz_header(); + SrsUniquePtr data(new uint8_t[nb_data]); + + SrsUniquePtr buffer(new SrsBuffer((char*)data.get(), nb_data)); + if ((err = mdat->encode(buffer.get())) != srs_success) { + return srs_error_wrap(err, "encode mdat"); + } + + // TODO: FIXME: Ensure all bytes are writen. + if ((err = writer_->write(data.get(), nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write mdat"); + } + + vector::iterator it; + // write video sample data + for (it = video_samples_->samples.begin(); it != video_samples_->samples.end(); ++it) { + SrsMp4Sample* sample = *it; + + // TODO: FIXME: Ensure all bytes are writen. + // TODO: do cbcs encryption here. sample are nalu_length + nalu data. + if ((err = writer_->write(sample->data, sample->nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write sample"); + } + } + + // write audio sample data + for (it = audio_samples_->samples.begin(); it != audio_samples_->samples.end(); ++it) { + SrsMp4Sample* sample = *it; + + // TODO: FIXME: Ensure all bytes are writen. + // TODO: do cbcs encryption here + if ((err = writer_->write(sample->data, sample->nb_data, NULL)) != srs_success) { + return srs_error_wrap(err, "write sample"); + } + } + } + + return err; +} diff --git a/trunk/src/kernel/srs_kernel_mp4.hpp b/trunk/src/kernel/srs_kernel_mp4.hpp index c4b1fdf53..134eaacb9 100644 --- a/trunk/src/kernel/srs_kernel_mp4.hpp +++ b/trunk/src/kernel/srs_kernel_mp4.hpp @@ -113,6 +113,27 @@ enum SrsMp4BoxType SrsMp4BoxTypeSIDX = 0x73696478, // 'sidx' SrsMp4BoxTypeHEV1 = 0x68657631, // 'hev1' SrsMp4BoxTypeHVCC = 0x68766343, // 'hvcC' + SrsMp4BoxTypeSENC = 0x73656e63, // 'senc' + SrsMp4BoxTypeSAIZ = 0x7361697a, // 'saiz' + SrsMp4BoxTypeSAIO = 0x7361696f, // 'saio' + SrsMp4BoxTypeENCV = 0x656e6376, // 'encv' + SrsMp4BoxTypeENCA = 0x656e6361, // 'enca' + SrsMp4BoxTypeSINF = 0x73696e66, // 'sinf' + SrsMp4BoxTypeSCHI = 0x73636869, // 'schi' + SrsMp4BoxTypeTENC = 0x74656e63, // 'tenc' + SrsMp4BoxTypeFRMA = 0x66726d61, // 'frma' + SrsMp4BoxTypeSCHM = 0x7363686d, // 'schm' +}; + +// Common encryption scheme types +// @see ISO-IEC-23001-7.pdf, 4.2 +enum SrsMp4CENSchemeType +{ + SrsMp4CENSchemeCENC = 0x63656e63, // 'cenc' + SrsMp4CENSchemeCBC1 = 0x63626331, // 'cbc1' + SrsMp4CENSchemeCENS = 0x63656e73, // 'cens' + SrsMp4CENSchemeCBCS = 0x63626373, // 'cbcs' + SrsMp4CENSchemeSVE1 = 0x73766531, // 'sve1' }; // 8.4.3.3 Semantics @@ -317,9 +338,9 @@ public: // Get the header of moof. virtual SrsMp4MovieFragmentHeaderBox* mfhd(); virtual void set_mfhd(SrsMp4MovieFragmentHeaderBox* v); - // Get the traf. - virtual SrsMp4TrackFragmentBox* traf(); - virtual void set_traf(SrsMp4TrackFragmentBox* v); + + // Let moof support more than one traf + virtual void add_traf(SrsMp4TrackFragmentBox* v); }; // 8.8.5 Movie Fragment Header Box (mfhd) @@ -499,7 +520,7 @@ class SrsMp4TrackFragmentRunBox : public SrsMp4FullBox public: // The number of samples being added in this run; also the number of rows in the following // table (the rows can be empty) - //uint32_t sample_count; + // uint32_t sample_count; // The following are optional fields public: // added to the implicit or explicit data_offset established in the track fragment header. @@ -710,8 +731,7 @@ public: virtual ~SrsMp4MovieExtendsBox(); public: // Get the track extends box. - virtual SrsMp4TrackExtendsBox* trex(); - virtual void set_trex(SrsMp4TrackExtendsBox* v); + virtual void add_trex(SrsMp4TrackExtendsBox* v); }; // 8.8.3 Track Extends Box(trex) @@ -1869,6 +1889,348 @@ public: virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); }; +// Sample auxiliary information sizes box (saiz) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.7.8, page 62 +// @see https://github.com/gpac/mp4box.js/blob/master/src/parsing/saiz.js +// Syntax +// aligned(8) class SampleAuxiliaryInformationSizesBox extends FullBox('saiz', version=0, flags) +// { +// if (flags & 1) { +// unsigned int(32) aux_info_type; +// unsigned int(32) aux_info_type_parameter; +// } +// unsigned int(8) default_sample_info_size; +// unsigned int(32) sample_count; +// if (default_sample_info_size == 0) { +// unsigned int(8) sample_info_size[sample_count]; +// } +// } +class SrsMp4SampleAuxiliaryInfoSizeBox: public SrsMp4FullBox +{ +public: + uint32_t aux_info_type; + uint32_t aux_info_type_parameter; + + uint8_t default_sample_info_size; + uint32_t sample_count; + std::vector sample_info_sizes; + +public: + SrsMp4SampleAuxiliaryInfoSizeBox(); + virtual ~SrsMp4SampleAuxiliaryInfoSizeBox(); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Sample auxiliary information offsets box (saio) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.7.9, page 63 +// @see https://github.com/gpac/mp4box.js/blob/master/src/parsing/saio.js +// Syntax +// aligned(8) class SampleAuxiliaryInformationOffsetsBox extends FullBox('saio', version, flags) +// { +// if (flags & 1) { +// unsigned int(32) aux_info_type; +// unsigned int(32) aux_info_type_parameter; +// } +// unsigned int(32) entry_count; +// if (version == 0) { +// unsigned int(32) offset[entry_count]; +// } else { +// unsigned int(64) offset[entry_count]; +// } +// } +class SrsMp4SampleAuxiliaryInfoOffsetBox: public SrsMp4FullBox +{ +public: + uint32_t aux_info_type; + uint32_t aux_info_type_parameter; + // uint32_t entry_count; + std::vector offsets; + +public: + SrsMp4SampleAuxiliaryInfoOffsetBox(); + virtual ~SrsMp4SampleAuxiliaryInfoOffsetBox(); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +enum SrsMp4CencSampleEncryptionFlags +{ + SrsMp4CencSampleEncryptionTrackDefault = 0x01, + SrsMp4CencSampleEncryptionUseSubSample = 0x02, +}; + +struct SrsMp4SubSampleEncryptionInfo : public ISrsCodec +{ + uint16_t bytes_of_clear_data; + uint32_t bytes_of_protected_data; + + SrsMp4SubSampleEncryptionInfo(); + virtual ~SrsMp4SubSampleEncryptionInfo(); + + virtual uint64_t nb_bytes(); + virtual srs_error_t encode(SrsBuffer* buf); + virtual srs_error_t decode(SrsBuffer* buf); + + virtual std::stringstream& dumps(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +class SrsMp4SampleEncryptionEntry : public ISrsCodec +{ +public: + // if flags && 0x02 + std::vector subsample_infos; + +public: + SrsMp4SampleEncryptionEntry(SrsMp4FullBox* senc, uint8_t per_sample_iv_size); + virtual ~SrsMp4SampleEncryptionEntry(); + + virtual srs_error_t set_iv(uint8_t* iv, uint8_t iv_size); + virtual uint64_t nb_bytes(); + virtual srs_error_t encode(SrsBuffer* buf); + virtual srs_error_t decode(SrsBuffer* buf); + + virtual std::stringstream& dumps(std::stringstream& ss, SrsMp4DumpContext dc); + +private: + SrsMp4FullBox* senc_; + uint8_t per_sample_iv_size_; + uint8_t* iv_; +}; + +// Sample encryption box (senc) +// @see ISO-IEC-23001-7.pdf 7.2.1 +// @see https://cdn.standards.iteh.ai/samples/84637/c960c91d60ae4da7a2f9380bd7e08642/ISO-IEC-FDIS-23001-7.pdf +// CENC SAI: sample auxiliary information associated with a sample and containing cryptographic information +// such as initialization vector or subsample information +// @see ISO-IEC-23001-7.pdf 7.2.2 +// Syntax +// aligned(8) class SampleEncryptionBox extend FullBox(`senc`, version=0, flags) +// { +// unsigned int(32) sample_count; +// { +// unsigned int(Per_Sample_IV_Size*8) InitializationVector; +// if (flags & 0x000002) +// { +// unsigned int(16) subsample_count; +// { +// unsigned int(16) BytesOfClearData; +// unsigned int(32) BytesOfProtectedData; +// } [ subsample_count ] +// } +// } [ sample_count ] +// } +class SrsMp4SampleEncryptionBox: public SrsMp4FullBox +{ +public: + std::vector entries; + +private: + uint8_t per_sample_iv_size_; + +public: + // @see ISO-IEC-23001-7.pdf 9.1 + // Per_Sample_IV_Size has supported values: 0, 8, 16. + SrsMp4SampleEncryptionBox(uint8_t per_sample_iv_size); + virtual ~SrsMp4SampleEncryptionBox(); +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Original Format Box (frma) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.2, page 81 +// aligned(8) class OriginalFormatBox(codingname) extends Box ('frma') { +// unsigned int(32) data_format = codingname; +// } +class SrsMp4OriginalFormatBox : public SrsMp4Box +{ +private: + uint32_t data_format_; + +public: + SrsMp4OriginalFormatBox(uint32_t original_format); + virtual ~SrsMp4OriginalFormatBox(); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Scheme Type Box (schm) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.5, page 81 +// aligned(8) class SchemeTypeBox extends FullBox('schm', 0, flags) { +// unsigned int(32) scheme_type; // 4CC identifying the scheme +// unsigned int(32) scheme_version; // scheme version +// if (flags & 0x000001) { +// unsigned int(8) scheme_uri[]; // browser uri +// } +// } +// @see @see ISO-IEC-23001-7.pdf 4.1 +// the scheme_version field SHALL be set to 0x00010000 (Major version 1, Minor version 0). +#define SCHM_SCHEME_URI_MAX_SIZE 128 +class SrsMp4SchemeTypeBox : public SrsMp4FullBox +{ +public: + uint32_t scheme_type; + uint32_t scheme_version; + char scheme_uri[SCHM_SCHEME_URI_MAX_SIZE]; + uint32_t scheme_uri_size; + +public: + SrsMp4SchemeTypeBox(); + virtual ~SrsMp4SchemeTypeBox(); + +public: + virtual void set_scheme_uri(char* uri, uint32_t uri_size); +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// Scheme Information Box (schi) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.6, page 82 +// aligned(8) class SchemeInformationBox extends Box('schi') { +// Box scheme_specific_data[]; +// } +class SrsMp4SchemeInfoBox : public SrsMp4Box +{ +public: + SrsMp4SchemeInfoBox(); + virtual ~SrsMp4SchemeInfoBox(); +}; + +// Protection Scheme Information Box (sinf) +// @see ISO_IEC_14496-12-base-format-2012.pdf, 8.12.1, page 80 +// aligned(8) class ProtectionSchemeInfoBox(fmt) extends Box('sinf') { +// OriginalFormatBox(fmt) original_format; // frma +// SchemeTypeBox scheme_type_box; // optional +// SchemeInformationBox info; // optional +// } +class SrsMp4ProtectionSchemeInfoBox : public SrsMp4Box +{ +public: + SrsMp4ProtectionSchemeInfoBox(); + virtual ~SrsMp4ProtectionSchemeInfoBox(); + +public: + // Get the Original Format Box (frma) + virtual SrsMp4OriginalFormatBox* frma(); + virtual void set_frma(SrsMp4OriginalFormatBox* v); + // Get the Scheme Type Box (schm) + virtual SrsMp4SchemeTypeBox* schm(); + virtual void set_schm(SrsMp4SchemeTypeBox* v); + // Get the Scheme Information Box (schi) + virtual SrsMp4SchemeInfoBox* schi(); + virtual void set_schi(SrsMp4SchemeInfoBox* v); +}; + +// Track Encryption box (tenc) +// @see ISO-IEC-23001-7.pdf 8.2 +// aligned(8) class TrackEncryptionBox extends FullBox('tenc', version, flags=0) { +// unsigned int(8) reserved = 0; +// if (version == 0) { +// unsigned int(8) reserved = 0; +// } else { // version is 1 or greater +// unsigned int(4) default_crypt_byte_block; +// unsigned int(4) default_skip_byte_block; +// } +// unsigned int(8) default_isProtected; +// unsigned int(8) default_Per_Sample_IV_Size; +// unsigned int(8)[16] default_KID; +// if (default_isProtected == 1 && default_Per_Sample_IV_Size == 0) { +// unsigned int(8) default_constant_IV_size; +// unsigned int(8)[default_constant_IV_size] default_constant_IV; +// } +// } +// @see https://developer.apple.com/documentation/http-live-streaming/about-the-common-media-application-format-with-http-live-streaming-hls +// For fragmented MPEG-4 Segments, an EXT-X-KEY tag with a METHOD=SAMPLE-AES attribute indicates that +// the Segment is encrypted using the `cbcs` scheme in ISO/IEC 23001-7. +// HLS supports unencrypted and encrypted with 'cbcs'. +// @see ISO-IEC-23001-7.pdf 10.4.1 Definition +// 'cbcs' AES-CBC subsample pattern encryption scheme. +// The 'scheme_type' field of the scheme Type Box('schm') SHALL be set to 'cbcs'. +// the version of the Track Encryption Box('tenc') SHALL be 1. +// Encrypted video tracks using NAL Structured Video conforming to ISO/IEC 14496-15 SHALL be +// protected using Subsample encryption specified in 9.5, and SHALL use pattern encryption as specified +// in 9.6. As a result, the fields crypt_byte_block and skip_byte_block SHALL NOT be 0. +// Constant IVs SHALL be used; 'default_Per_Sample_IV_Size' and 'Per_Sample_IV_Size', SHALL be 0. +// Tracks other than video are protected using whole-block full-sample encryption as specified in 9.7 and +// hence skip_byte_block SHALL be 0. +// Pattern Block length, i.e. crypt_byte_block + skip_byte_block SHOULD equal 10. +// For all video NAL units, including in 'avc1', the slice header SHALL be unencrypted. +// The first complete byte of video slice data(following the video slice header) SHALL begin a single +// Subsample protected byte range indicated by the start of BytesOfProtectedData, which extends to +// the end of the video NAL. +// NOTE 1 For AVC VCL NAL units, the encryption pattern starts at an offset rounded to the next byte after +// the slice header, i.e. on the first full byte of slice data. For HEVC, the encryption pattern starts after +// the byte_alignment() field that terminates the slice_segment_header(), i.e. on the first byte of slice data. +// +// @see ISO-IEC-23001-7.pdf 10.4.2 'cbcs' AES-CBC mode pattern encryption scheme application(informative) +// An encrypt:skip pattern of 1:9(i.e. 10% partial encryption) is recommended. Even though the syntax +// allows many different encryption patterns, a pattern of ten Blocks is recommended. This means that the +// skipped Blocks will be (10-N). The number of encrypted cipher blocks N can span multiple contiguous +// 16-byte Blocks(e.g. three encrypted Blocks followed by seven unencrypted Blocks would result in 30% +// partial encryption of the video data). +// For example, to achieve 10 % encryption, the first Block of the pattern is encrypted and the following +// nine Blocks are left unencrypted. The pattern is repeated every 160 bytes of the protected range, until +// the end of the range. If the protected range of the slice body is not a multiple of the pattern length +// (e.g. 160 bytes), then the pattern sequence applies to the included whole 16-byte Blocks and a partial +// 16-byte Block that may remain where the pattern is terminated by the byte length of the range +// BytesOfProtectedData, is left unencrypted. +// +// @see ISO-IEC-23001-7.pdf 9.7 Whole-block full sample encryption +// In whole-block full sample encryption, the entire sample is protected. Every sample is encrypted +// starting at offset 0(there is no unprotected preamble) up to the last 16-byte boundary, leaving any +// trailing 0-15 bytes in the clear. The IV is reset at every sample. +class SrsMp4TrackEncryptionBox : public SrsMp4FullBox +{ +public: + uint8_t reserved; + uint8_t reserved_2; + uint8_t default_crypt_byte_block; + uint8_t default_skip_byte_block; + uint8_t default_is_protected; + uint8_t default_per_sample_IV_size; + uint8_t default_KID[16]; + uint8_t default_constant_IV_size; + uint8_t default_constant_IV[16]; +public: + SrsMp4TrackEncryptionBox(); + virtual ~SrsMp4TrackEncryptionBox(); + +public: + virtual void set_default_constant_IV(uint8_t* iv, uint8_t iv_size); + +protected: + virtual int nb_header(); + virtual srs_error_t encode_header(SrsBuffer* buf); + virtual srs_error_t decode_header(SrsBuffer* buf); +public: + virtual std::stringstream& dumps_detail(std::stringstream& ss, SrsMp4DumpContext dc); +}; + +// TODO: add SchemeTypeBox(schm), set scheme_type=cbcs + // Generally, a MP4 sample contains a frame, for example, a video frame or audio frame. class SrsMp4Sample { @@ -1931,7 +2293,7 @@ public: virtual srs_error_t write(SrsMp4MovieBox* moov); // Write the samples info to moof. // @param The dts is the dts of last segment. - virtual srs_error_t write(SrsMp4MovieFragmentBox* moof, uint64_t dts); + virtual srs_error_t write(SrsMp4TrackFragmentBox* traf, uint64_t dts); private: virtual srs_error_t write_track(SrsFrameType track, SrsMp4DecodingTime2SampleBox* stts, SrsMp4SyncSampleBox* stss, SrsMp4CompositionTime2SampleBox* ctts, @@ -2114,22 +2476,67 @@ private: }; // A fMP4 encoder, to write the init.mp4 with sequence header. +// TODO: What the M2ts short for? class SrsMp4M2tsInitEncoder { private: ISrsWriter* writer; + +private: + uint8_t crypt_byte_block_; + uint8_t skip_byte_block_; + unsigned char kid_[16]; + unsigned char iv_[16]; + uint8_t iv_size_; + bool is_protected_; + public: SrsMp4M2tsInitEncoder(); virtual ~SrsMp4M2tsInitEncoder(); public: // Initialize the encoder with a writer w. virtual srs_error_t initialize(ISrsWriter* w); + // set encryption + // TODO: review kid(map to a key) and iv, which are shared between audio/video tracks. + virtual void config_encryption(uint8_t crypt_byte_block, uint8_t skip_byte_block, unsigned char* kid, unsigned char* iv, uint8_t iv_size); // Write the sequence header. + // TODO: merge this method to its sibling. virtual srs_error_t write(SrsFormat* format, bool video, int tid); + + /** + * The mp4 box format for init.mp4. + * + * |ftyp| + * |moov| + * | |mvhd| + * | |trak| + * | |trak| + * | |....| + * | |mvex| + * | | |trex| + * | | |trex| + * | | |....| + * + * Write the sequence header with both video and audio track. + */ + virtual srs_error_t write(SrsFormat* format, int v_tid, int a_tid); + +private: + /** + * box->type = 'encv' or 'enca' + * |encv| + * | |sinf| + * | | |frma| + * | | |schm| + * | | |schi| + * | | | |tenc| + */ + virtual srs_error_t config_sample_description_encryption(SrsMp4SampleEntry* box); }; // A fMP4 encoder, to cache segments then flush to disk, because the fMP4 should write // trun box before mdat. +// TODO: fmp4 support package more than one tracks. class SrsMp4M2tsSegmentEncoder { private: @@ -2163,6 +2570,54 @@ public: virtual srs_error_t flush(uint64_t& dts); }; +// A fMP4 encoder, to cache segments then flush to disk, because the fMP4 should write +// trun box before mdat. +// TODO: fmp4 support package more than one tracks. +class SrsFmp4SegmentEncoder +{ +private: + ISrsWriter* writer_; + uint32_t sequence_number_; + // TODO: audio, video may have different basetime. + srs_utime_t decode_basetime_; + uint32_t audio_track_id_; + uint32_t video_track_id_; +private: + uint32_t nb_audios_; + uint32_t nb_videos_; + uint32_t styp_bytes_; + uint64_t mdat_audio_bytes_; + uint64_t mdat_video_bytes_; + SrsMp4SampleManager* audio_samples_; + SrsMp4SampleManager* video_samples_; +private: + // Encryption + unsigned char* key_; + unsigned char iv_[16]; + bool do_sample_encryption_; +public: + SrsFmp4SegmentEncoder(); + virtual ~SrsFmp4SegmentEncoder(); +public: + // Initialize the encoder with a writer w. + virtual srs_error_t initialize(ISrsWriter* w, uint32_t sequence, srs_utime_t basetime, uint32_t v_tid, uint32_t a_tid); + // config cipher + virtual srs_error_t config_cipher(unsigned char* key, unsigned char* iv); + // Cache a sample. + // @param ht, The sample handler type, audio/soun or video/vide. + // @param ft, The frame type. For video, it's SrsVideoAvcFrameType. + // @param dts The output dts in milliseconds. + // @param pts The output pts in milliseconds. + // @param sample The output payload, user must free it. + // @param nb_sample The output size of payload. + // @remark All samples are RAW AAC/AVC data, because sequence header is writen to init.mp4. + virtual srs_error_t write_sample(SrsMp4HandlerType ht, uint16_t ft, + uint32_t dts, uint32_t pts, uint8_t* sample, uint32_t nb_sample); + // Flush the encoder, to write the moof and mdat. + virtual srs_error_t flush(uint64_t dts); +}; + + // LCOV_EXCL_START ///////////////////////////////////////////////////////////////////////////////// // MP4 dumps functions. diff --git a/trunk/src/protocol/srs_protocol_http_stack.cpp b/trunk/src/protocol/srs_protocol_http_stack.cpp index d6c2d2907..2617bacc0 100644 --- a/trunk/src/protocol/srs_protocol_http_stack.cpp +++ b/trunk/src/protocol/srs_protocol_http_stack.cpp @@ -391,6 +391,7 @@ srs_error_t SrsHttpFileServer::serve_http(ISrsHttpResponseWriter* w, ISrsHttpMes string upath = r->path(); string fullpath = srs_http_fs_fullpath(dir, entry->pattern, upath); + string basename = srs_path_basename(upath); // stat current dir, if exists, return error. if (!_srs_path_exists(fullpath)) { @@ -400,18 +401,18 @@ srs_error_t SrsHttpFileServer::serve_http(ISrsHttpResponseWriter* w, ISrsHttpMes } srs_trace("http match file=%s, pattern=%s, upath=%s", fullpath.c_str(), entry->pattern.c_str(), upath.c_str()); - + // handle file according to its extension. // use vod stream for .flv/.fhv - if (srs_string_ends_with(fullpath, ".flv") || srs_string_ends_with(fullpath, ".fhv")) { + if (srs_string_ends_with(upath, ".flv", ".fhv")) { return serve_flv_file(w, r, fullpath); - } else if (srs_string_ends_with(fullpath, ".mp4")) { - return serve_mp4_file(w, r, fullpath); } else if (srs_string_ends_with(upath, ".m3u8")) { - return serve_m3u8_file(w, r, fullpath); - } else if (srs_string_ends_with(upath, ".ts")) { - return serve_ts_file(w, r, fullpath); - } + return serve_m3u8_ctx(w, r, fullpath); + } else if (srs_string_ends_with(upath, ".ts", ".m4s") || basename == "init.mp4") { + return serve_ts_ctx(w, r, fullpath); + } else if (srs_string_ends_with(upath, ".mp4")) { + return serve_mp4_file(w, r, fullpath); + } // serve common static file. return serve_file(w, r, fullpath); @@ -553,16 +554,6 @@ srs_error_t SrsHttpFileServer::serve_mp4_file(ISrsHttpResponseWriter* w, ISrsHtt return serve_mp4_stream(w, r, fullpath, start, end); } -srs_error_t SrsHttpFileServer::serve_m3u8_file(ISrsHttpResponseWriter * w, ISrsHttpMessage * r, std::string fullpath) -{ - return serve_m3u8_ctx(w, r, fullpath); -} - -srs_error_t SrsHttpFileServer::serve_ts_file(ISrsHttpResponseWriter * w, ISrsHttpMessage * r, std::string fullpath) -{ - return serve_ts_ctx(w, r, fullpath); -} - srs_error_t SrsHttpFileServer::serve_flv_stream(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, string fullpath, int64_t offset) { // @remark For common http file server, we don't support stream request, please use SrsVodStream instead. diff --git a/trunk/src/protocol/srs_protocol_http_stack.hpp b/trunk/src/protocol/srs_protocol_http_stack.hpp index ce25dae2e..fa6b8222e 100644 --- a/trunk/src/protocol/srs_protocol_http_stack.hpp +++ b/trunk/src/protocol/srs_protocol_http_stack.hpp @@ -351,8 +351,6 @@ private: virtual srs_error_t serve_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); virtual srs_error_t serve_flv_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); virtual srs_error_t serve_mp4_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); - virtual srs_error_t serve_m3u8_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); - virtual srs_error_t serve_ts_file(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); protected: // When access flv file with x.flv?start=xxx virtual srs_error_t serve_flv_stream(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath, int64_t offset); @@ -371,6 +369,7 @@ protected: // Remark 2: // If use two same "hls_ctx" in different requests, SRS cannot detect so that they will be treated as one. virtual srs_error_t serve_m3u8_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); + // the ts file including: .ts .m4s init.mp4 virtual srs_error_t serve_ts_ctx(ISrsHttpResponseWriter* w, ISrsHttpMessage* r, std::string fullpath); protected: // Copy the fs to response writer in size bytes. diff --git a/trunk/src/utest/srs_utest_config.cpp b/trunk/src/utest/srs_utest_config.cpp index 9b06788ee..a2fe003f5 100644 --- a/trunk/src/utest/srs_utest_config.cpp +++ b/trunk/src/utest/srs_utest_config.cpp @@ -3732,12 +3732,15 @@ VOID TEST(ConfigMainTest, CheckVhostConfig5) if (true) { MockSrsConfig conf; - HELPER_ASSERT_SUCCESS(conf.parse(_MIN_OK_CONF "vhost ossrs.net{hls{hls_keys on;hls_fragments_per_key 5;hls_key_file xxx;hls_key_file_path xxx2;hls_key_url xxx3;}}")); + HELPER_ASSERT_SUCCESS(conf.parse(_MIN_OK_CONF "vhost ossrs.net{hls{hls_keys on;hls_fragments_per_key 5;hls_key_file xxx;hls_key_file_path xxx2;hls_key_url xxx3;hls_use_fmp4 on;hls_fmp4_file xx.m4s;hls_init_file yy-init.mp4;}}")); EXPECT_TRUE(conf.get_hls_keys("ossrs.net")); EXPECT_EQ(5, conf.get_hls_fragments_per_key("ossrs.net")); EXPECT_STREQ("xxx", conf.get_hls_key_file("ossrs.net").c_str()); EXPECT_STREQ("xxx2", conf.get_hls_key_file_path("ossrs.net").c_str()); EXPECT_STREQ("xxx3", conf.get_hls_key_url("ossrs.net").c_str()); + EXPECT_TRUE(conf.get_hls_use_fmp4("ossrs.net")); + EXPECT_STREQ("xx.m4s", conf.get_hls_fmp4_file("ossrs.net").c_str()); + EXPECT_STREQ("yy-init.mp4", conf.get_hls_init_file("ossrs.net").c_str()); } if (true) { @@ -5125,6 +5128,27 @@ VOID TEST(ConfigEnvTest, CheckEnvValuesHls) SrsSetEnvConfig(conf, hls_dts_directly, "SRS_VHOST_HLS_HLS_DTS_DIRECTLY", "off"); EXPECT_FALSE(conf.get_vhost_hls_dts_directly("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_use_fmp4_on, "SRS_VHOST_HLS_HLS_USE_FMP4", "on"); + EXPECT_TRUE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_use_fmp4_off, "SRS_VHOST_HLS_HLS_USE_FMP4", "off"); + EXPECT_FALSE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_use_fmp4_unexpected, "SRS_VHOST_HLS_HLS_USE_FMP4", "xx"); + EXPECT_FALSE(conf.get_hls_use_fmp4("__defaultVhost__")); + + SrsSetEnvConfig(conf, hls_fmp4_file, "SRS_VHOST_HLS_HLS_FMP4_FILE", "xxx.m4s"); + EXPECT_STREQ("xxx.m4s", conf.get_hls_fmp4_file("__defaultVhost__").c_str()); + + SrsSetEnvConfig(conf, hls_init_file, "SRS_VHOST_HLS_HLS_INIT_FILE", "yyy-init.mp4"); + EXPECT_STREQ("yyy-init.mp4", conf.get_hls_init_file("__defaultVhost__").c_str()); + } + + // Test default value for hls_init_file with a fresh config + { + MockSrsConfig conf; + EXPECT_STREQ("[app]/[stream]/init.mp4", conf.get_hls_init_file("__defaultVhost__").c_str()); } } diff --git a/trunk/src/utest/srs_utest_fmp4.cpp b/trunk/src/utest/srs_utest_fmp4.cpp new file mode 100644 index 000000000..0aef62c17 --- /dev/null +++ b/trunk/src/utest/srs_utest_fmp4.cpp @@ -0,0 +1,810 @@ +// +// Copyright (c) 2013-2025 The SRS Authors +// +// SPDX-License-Identifier: MIT +// + +#include + +#include +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include + +// Mock classes for testing +class MockSrsRequest : public SrsRequest +{ +public: + MockSrsRequest() { + vhost = "__defaultVhost__"; + app = "live"; + stream = "livestream"; + } + virtual ~MockSrsRequest() {} +}; + +class MockSrsFormat : public SrsFormat +{ +public: + MockSrsFormat() { + initialize(); + + // Setup video sequence header (H.264 AVC) + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + on_video(0, (char*)video_raw, sizeof(video_raw)); + + // Setup audio sequence header (AAC) + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + on_audio(0, (char*)audio_raw, sizeof(audio_raw)); + } + virtual ~MockSrsFormat() {} +}; + +class MockSrsSharedPtrMessage : public SrsSharedPtrMessage +{ +public: + MockSrsSharedPtrMessage(bool is_video_msg, uint32_t ts) { + timestamp = ts; + + // Create sample payload + char* payload = new char[1024]; + memset(payload, 0x00, 1024); + SrsSharedPtrMessage::wrap(payload, 1024); + + if (is_video_msg) { + ptr->header.message_type = RTMP_MSG_VideoMessage; + } else { + ptr->header.message_type = RTMP_MSG_AudioMessage; + } + } + virtual ~MockSrsSharedPtrMessage() {} +}; + +VOID TEST(Fmp4Test, SrsInitMp4Segment_VideoOnly) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_video.mp4"); + MockSrsFormat fmt; + + HELPER_ASSERT_SUCCESS(segment.write_video_only(&fmt, 1)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify the file contains expected MP4 boxes + string content = fw.str(); + EXPECT_TRUE(content.find("ftyp") != string::npos); + EXPECT_TRUE(content.find("moov") != string::npos); +} + +VOID TEST(Fmp4Test, SrsInitMp4Segment_AudioOnly) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_audio.mp4"); + MockSrsFormat fmt; + + HELPER_ASSERT_SUCCESS(segment.write_audio_only(&fmt, 2)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify the file contains expected MP4 boxes + string content = fw.str(); + EXPECT_TRUE(content.find("ftyp") != string::npos); + EXPECT_TRUE(content.find("moov") != string::npos); +} + +VOID TEST(Fmp4Test, SrsInitMp4Segment_AudioVideo) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_av.mp4"); + MockSrsFormat fmt; + + HELPER_ASSERT_SUCCESS(segment.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify the file contains expected MP4 boxes + string content = fw.str(); + EXPECT_TRUE(content.find("ftyp") != string::npos); + EXPECT_TRUE(content.find("moov") != string::npos); +} + +VOID TEST(Fmp4Test, SrsInitMp4Segment_WithEncryption) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init_encrypted.mp4"); + MockSrsFormat fmt; + + // Configure encryption + unsigned char kid[16] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, + 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}; + unsigned char iv[16] = {0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, + 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 0x20}; + + HELPER_ASSERT_SUCCESS(segment.config_cipher(kid, iv, 16)); + HELPER_ASSERT_SUCCESS(segment.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); +} + +VOID TEST(Fmp4Test, SrsHlsM4sSegment_Basic) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsHlsM4sSegment segment(&fw); + + // Initialize segment with a path that doesn't require file system operations + HELPER_ASSERT_SUCCESS(segment.initialize(0, 1, 2, 100, "segment-100.m4s")); + + // Write video sample + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_msg(true, 1000); + HELPER_ASSERT_SUCCESS(segment.write(&video_msg, &fmt)); + + // Write audio sample + MockSrsSharedPtrMessage audio_msg(false, 2000); // Different timestamp + HELPER_ASSERT_SUCCESS(segment.write(&audio_msg, &fmt)); + + // Test duration - should be > 0 after writing samples with different timestamps + EXPECT_GT(segment.duration(), 0); +} + +VOID TEST(Fmp4Test, SrsHlsM4sSegment_WithEncryption) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsHlsM4sSegment segment(&fw); + + // Initialize segment + HELPER_ASSERT_SUCCESS(segment.initialize(0, 1, 2, 101, "segment-101.m4s")); + + // Configure encryption + unsigned char key[16] = {0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, + 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30}; + unsigned char iv[16] = {0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, + 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40}; + segment.config_cipher(key, iv); + + // Verify IV is stored + EXPECT_EQ(0x31, segment.iv[0]); + EXPECT_EQ(0x40, segment.iv[15]); + + // Write samples with different timestamps to create duration + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_msg1(true, 1000); + HELPER_ASSERT_SUCCESS(segment.write(&video_msg1, &fmt)); + + MockSrsSharedPtrMessage video_msg2(true, 2000); + HELPER_ASSERT_SUCCESS(segment.write(&video_msg2, &fmt)); + + // Test that segment has content + EXPECT_GT(segment.duration(), 0); +} + +VOID TEST(Fmp4Test, SrsFmp4SegmentEncoder_Basic) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Initialize encoder + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + // Write video sample + uint8_t video_sample[] = {0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x20}; + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + 1000, 1000, video_sample, sizeof(video_sample))); + + // Write audio sample + uint8_t audio_sample[] = {0xff, 0xf1, 0x50, 0x80, 0x01, 0x3f, 0xfc}; + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeSOUN, 0x00, + 1000, 1000, audio_sample, sizeof(audio_sample))); + + // Flush to file + HELPER_ASSERT_SUCCESS(encoder.flush(2000)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify basic structure (content may be binary, so just check size) + EXPECT_GT(fw.filesize(), 100); // Should have reasonable size for fMP4 structure +} + +VOID TEST(Fmp4Test, SrsFmp4SegmentEncoder_WithEncryption) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Initialize encoder + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + // Configure encryption + unsigned char key[16] = {0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, + 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50}; + unsigned char iv[16] = {0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, + 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 0x60}; + encoder.config_cipher(key, iv); + + // Write encrypted samples + uint8_t video_sample[] = {0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x20}; + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + 1000, 1000, video_sample, sizeof(video_sample))); + + HELPER_ASSERT_SUCCESS(encoder.flush(2000)); + EXPECT_TRUE(fw.filesize() > 0); + + // Verify encryption boxes are present + string content = fw.str(); + EXPECT_TRUE(content.find("senc") != string::npos); // Sample Encryption Box +} + +VOID TEST(Fmp4Test, SrsHlsFmp4Muxer_Basic) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + + // Initialize muxer + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + // Test basic properties + EXPECT_EQ(0, muxer.sequence_no()); + EXPECT_EQ(0, (int)muxer.duration()); + EXPECT_EQ(0, muxer.deviation()); + + // Test codec management + muxer.set_latest_acodec(SrsAudioCodecIdAAC); + muxer.set_latest_vcodec(SrsVideoCodecIdAVC); + EXPECT_EQ(SrsAudioCodecIdAAC, muxer.latest_acodec()); + EXPECT_EQ(SrsVideoCodecIdAVC, muxer.latest_vcodec()); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsFmp4Muxer_WriteInitMp4) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(muxer.on_publish(&req)); + HELPER_ASSERT_SUCCESS(muxer.update_config(&req)); + + // Write init.mp4 with both audio and video + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, true)); + + // Write init.mp4 with video only + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, false)); + + // Write init.mp4 with audio only + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, false, true)); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsFmp4Muxer_WriteMedia) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(muxer.on_publish(&req)); + HELPER_ASSERT_SUCCESS(muxer.update_config(&req)); + + // Write init.mp4 first + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, true)); + + // Write video samples + MockSrsSharedPtrMessage video_msg(true, 1000); + HELPER_ASSERT_SUCCESS(muxer.write_video(&video_msg, &fmt)); + + // Write audio samples + MockSrsSharedPtrMessage audio_msg(false, 1000); + HELPER_ASSERT_SUCCESS(muxer.write_audio(&audio_msg, &fmt)); + + // Write more samples with time progression to accumulate duration + for (int i = 1; i <= 5; i++) { + MockSrsSharedPtrMessage video_msg2(true, 1000 + i * 1000); // 1 second increments + HELPER_ASSERT_SUCCESS(muxer.write_video(&video_msg2, &fmt)); + + MockSrsSharedPtrMessage audio_msg2(false, 1000 + i * 1000); + HELPER_ASSERT_SUCCESS(muxer.write_audio(&audio_msg2, &fmt)); + } + + // Should have accumulated duration from timestamp progression + EXPECT_GT(muxer.duration(), 0); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsMp4Controller_Basic) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + + // Initialize controller + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Test basic properties + EXPECT_EQ(0, controller.sequence_no()); + EXPECT_EQ(0, (int)controller.duration()); + + // Test URL generation (should return .m4s URL) + string url = controller.ts_url(); + EXPECT_TRUE(url.find(".m4s") != string::npos || url.empty()); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, SrsHlsMp4Controller_PublishWorkflow) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Publish stream + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + + // Handle sequence headers + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_sh(true, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&video_sh, &fmt)); + + MockSrsSharedPtrMessage audio_sh(false, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&audio_sh, &fmt)); + + // Write media samples + MockSrsSharedPtrMessage video_msg(true, 1000); + HELPER_ASSERT_SUCCESS(controller.write_video(&video_msg, &fmt)); + + MockSrsSharedPtrMessage audio_msg(false, 1000); + HELPER_ASSERT_SUCCESS(controller.write_audio(&audio_msg, &fmt)); + + // Unpublish + HELPER_ASSERT_SUCCESS(controller.on_unpublish()); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, SrsMp4TrackEncryptionBox_CBCS) +{ + srs_error_t err; + + SrsMp4TrackEncryptionBox tenc; + + // Configure for CBCS video encryption (1:9 pattern) + tenc.version = 1; + tenc.default_crypt_byte_block = 1; // Encrypt 1 block + tenc.default_skip_byte_block = 9; // Skip 9 blocks + tenc.default_is_protected = 1; + tenc.default_per_sample_IV_size = 0; // Use constant IV + tenc.default_constant_IV_size = 16; + + // Set Key ID + unsigned char kid[16] = {0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, + 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70}; + memcpy(tenc.default_KID, kid, 16); + + // Set constant IV + unsigned char iv[16] = {0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 0x80}; + tenc.set_default_constant_IV(iv, 16); + + // Verify configuration + EXPECT_EQ(1, tenc.version); + EXPECT_EQ(1, tenc.default_crypt_byte_block); + EXPECT_EQ(9, tenc.default_skip_byte_block); + EXPECT_EQ(1, tenc.default_is_protected); + EXPECT_EQ(0, tenc.default_per_sample_IV_size); + EXPECT_EQ(16, tenc.default_constant_IV_size); + EXPECT_EQ(0x61, tenc.default_KID[0]); + EXPECT_EQ(0x70, tenc.default_KID[15]); + EXPECT_EQ(0x71, tenc.default_constant_IV[0]); + EXPECT_EQ(0x80, tenc.default_constant_IV[15]); + + // Test encoding/decoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(tenc.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps - just verify it doesn't crash and produces some output + stringstream ss; + SrsMp4DumpContext dc; + tenc.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_FALSE(detail.empty()); // Should produce some output +} + +VOID TEST(Fmp4Test, SrsMp4TrackEncryptionBox_AudioFullSample) +{ + SrsMp4TrackEncryptionBox tenc; + + // Configure for audio full-sample encryption + tenc.version = 1; + tenc.default_crypt_byte_block = 0; // No pattern (full encryption) + tenc.default_skip_byte_block = 0; // No skip + tenc.default_is_protected = 1; + tenc.default_per_sample_IV_size = 0; // Use constant IV + tenc.default_constant_IV_size = 16; + + // Verify audio encryption configuration + EXPECT_EQ(0, tenc.default_crypt_byte_block); + EXPECT_EQ(0, tenc.default_skip_byte_block); + EXPECT_EQ(1, tenc.default_is_protected); +} + +VOID TEST(Fmp4Test, SrsMp4SampleEncryptionBox_Basic) +{ + srs_error_t err; + + SrsMp4SampleEncryptionBox senc(16); // 16-byte IV size + + // Test basic properties - flags may be set by constructor + EXPECT_EQ(0, senc.version); + EXPECT_EQ(0, (int)senc.entries.size()); + + // Test encoding empty box + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(senc.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps + stringstream ss; + SrsMp4DumpContext dc; + senc.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_TRUE(detail.find("sample_count=0") != string::npos); +} + +VOID TEST(Fmp4Test, SrsMp4SampleAuxiliaryInfoSizeBox_Basic) +{ + srs_error_t err; + + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + // Configure SAIZ box + saiz.version = 0; + saiz.flags = 0; + saiz.default_sample_info_size = 16; // 16-byte IV + saiz.sample_count = 0; + + // Test encoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(saiz.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_TRUE(detail.find("sample_count=0") != string::npos); +} + +VOID TEST(Fmp4Test, SrsMp4SampleAuxiliaryInfoOffsetBox_Basic) +{ + srs_error_t err; + + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + // Configure SAIO box + saio.version = 0; + saio.flags = 0; + saio.offsets.push_back(100); // Offset to SENC box + + // Test encoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(saio.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_TRUE(detail.find("entry_count=1") != string::npos); +} + +VOID TEST(Fmp4Test, SrsMp4OriginalFormatBox_Basic) +{ + srs_error_t err; + + SrsMp4OriginalFormatBox frma(SrsMp4BoxTypeAVC1); + + // Test encoding + char buffer_data[1024]; + SrsBuffer buf(buffer_data, 1024); + HELPER_ASSERT_SUCCESS(frma.encode(&buf)); + EXPECT_TRUE(buf.pos() > 0); + + // Test dumps - just verify it produces output + stringstream ss; + SrsMp4DumpContext dc; + frma.dumps_detail(ss, dc); + string detail = ss.str(); + EXPECT_FALSE(detail.empty()); +} + +VOID TEST(Fmp4Test, Integration_FullEncryptionWorkflow) +{ + srs_error_t err; + + // Test complete encryption workflow from init.mp4 to encrypted segments + + // 1. Create encrypted init.mp4 + MockSrsFileWriter init_fw; + SrsInitMp4Segment init_segment(&init_fw); + init_segment.set_path("encrypted_init.mp4"); + + unsigned char kid[16] = {0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, + 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90}; + unsigned char iv[16] = {0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, + 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0}; + + HELPER_ASSERT_SUCCESS(init_segment.config_cipher(kid, iv, 16)); + + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(init_segment.write(&fmt, 1, 2)); + EXPECT_TRUE(init_fw.filesize() > 0); + + // 2. Create encrypted segment + MockSrsFileWriter seg_fw; + SrsHlsM4sSegment m4s_segment(&seg_fw); + HELPER_ASSERT_SUCCESS(m4s_segment.initialize(0, 1, 2, 200, "encrypted-200.m4s")); + + unsigned char seg_key[16] = {0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, + 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 0xb0}; + unsigned char seg_iv[16] = {0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, + 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0}; + m4s_segment.config_cipher(seg_key, seg_iv); + + // Write samples to encrypted segment with time progression + MockSrsSharedPtrMessage video_msg1(true, 2000); + HELPER_ASSERT_SUCCESS(m4s_segment.write(&video_msg1, &fmt)); + + MockSrsSharedPtrMessage audio_msg1(false, 2500); + HELPER_ASSERT_SUCCESS(m4s_segment.write(&audio_msg1, &fmt)); + + MockSrsSharedPtrMessage video_msg2(true, 3000); + HELPER_ASSERT_SUCCESS(m4s_segment.write(&video_msg2, &fmt)); + + // Should have duration from timestamp progression + EXPECT_GT(m4s_segment.duration(), 0); + + // 3. Verify both files have content (encryption metadata is binary) + EXPECT_TRUE(init_fw.filesize() > 0); + EXPECT_GT(m4s_segment.duration(), 0); +} + +VOID TEST(Fmp4Test, EdgeCase_LargeTimestamp) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Test with large timestamp values + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + uint64_t large_timestamp = 0xFFFFFFFF; // Max 32-bit value + uint8_t sample[] = {0x00, 0x01, 0x02, 0x03}; + + HELPER_ASSERT_SUCCESS(encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + large_timestamp, large_timestamp, sample, sizeof(sample))); + + HELPER_ASSERT_SUCCESS(encoder.flush(large_timestamp + 1000)); + EXPECT_TRUE(fw.filesize() > 0); +} + +VOID TEST(Fmp4Test, ErrorHandling_InvalidTrackId) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + // Test with track ID 0 (may or may not be invalid depending on implementation) + err = encoder.initialize(&fw, 0, 0, 0, 0); + // Just verify the function can be called without crashing + if (err != srs_success) { + srs_freep(err); + } +} + +VOID TEST(Fmp4Test, ErrorHandling_NullWriter) +{ + srs_error_t err; + + SrsFmp4SegmentEncoder encoder; + + // Initialize with null writer - may or may not fail depending on implementation + err = encoder.initialize(NULL, 0, 0, 1, 2); + if (err != srs_success) { + srs_freep(err); + } + // Test passes if no crash occurs +} + +VOID TEST(Fmp4Test, ErrorHandling_WriteBeforeInitialize) +{ + srs_error_t err; + + SrsFmp4SegmentEncoder encoder; + uint8_t sample[] = {0x00, 0x01}; + + // Try to write sample before initialization - may or may not fail + err = encoder.write_sample(SrsMp4HandlerTypeVIDE, SrsVideoAvcFrameTypeKeyFrame, + 1000, 1000, sample, sizeof(sample)); + if (err != srs_success) { + srs_freep(err); + } + // Test passes if no crash occurs +} + +VOID TEST(Fmp4Test, ErrorHandling_FlushBeforeWrite) +{ + srs_error_t err; + + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + + HELPER_ASSERT_SUCCESS(encoder.initialize(&fw, 0, 0, 1, 2)); + + // Flush without writing any samples - may fail due to empty samples + err = encoder.flush(1000); + if (err != srs_success) { + srs_freep(err); + // This is expected behavior for empty flush + } +} + +VOID TEST(Fmp4Test, Configuration_TrackIdManagement) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Verify default track IDs + EXPECT_EQ(1, controller.video_track_id_); + EXPECT_EQ(2, controller.audio_track_id_); + + // Test sequence header tracking + EXPECT_FALSE(controller.has_video_sh_); + EXPECT_FALSE(controller.has_audio_sh_); + + // Set request first + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_sh(true, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&video_sh, &fmt)); + EXPECT_TRUE(controller.has_video_sh_); + + MockSrsSharedPtrMessage audio_sh(false, 0); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&audio_sh, &fmt)); + EXPECT_TRUE(controller.has_audio_sh_); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, Configuration_SequenceHeaderValidation) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + // Test sequence header without request (should fail) + MockSrsFormat fmt; + MockSrsSharedPtrMessage video_sh(true, 0); + HELPER_EXPECT_FAILED(controller.on_sequence_header(&video_sh, &fmt)); + + // Set request and try again + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + HELPER_ASSERT_SUCCESS(controller.on_sequence_header(&video_sh, &fmt)); + + controller.dispose(); +} + +VOID TEST(Fmp4Test, Performance_MultipleSegments) +{ + srs_error_t err; + + SrsHlsFmp4Muxer muxer; + HELPER_ASSERT_SUCCESS(muxer.initialize(1, 2)); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(muxer.on_publish(&req)); + HELPER_ASSERT_SUCCESS(muxer.update_config(&req)); + + MockSrsFormat fmt; + HELPER_ASSERT_SUCCESS(muxer.write_init_mp4(&fmt, true, true)); + + int initial_seq = muxer.sequence_no(); + + // Write many samples to create multiple segments + for (int i = 0; i < 500; i++) { + MockSrsSharedPtrMessage video_msg(true, i * 40); + HELPER_ASSERT_SUCCESS(muxer.write_video(&video_msg, &fmt)); + + if (i % 2 == 0) { // Write audio less frequently + MockSrsSharedPtrMessage audio_msg(false, i * 40); + HELPER_ASSERT_SUCCESS(muxer.write_audio(&audio_msg, &fmt)); + } + } + + // Should have created multiple segments + EXPECT_GT(muxer.sequence_no(), initial_seq); + EXPECT_GT(muxer.duration(), 0); + + muxer.dispose(); +} + +VOID TEST(Fmp4Test, Compatibility_SequenceHeaderIgnore) +{ + srs_error_t err; + + SrsHlsMp4Controller controller; + HELPER_ASSERT_SUCCESS(controller.initialize()); + + MockSrsRequest req; + HELPER_ASSERT_SUCCESS(controller.on_publish(&req)); + + MockSrsFormat fmt; + + // Create audio sequence header message + MockSrsSharedPtrMessage audio_sh(false, 0); + + // Should ignore sequence headers in write_audio + HELPER_ASSERT_SUCCESS(controller.write_audio(&audio_sh, &fmt)); + + // Regular audio message should be processed + MockSrsSharedPtrMessage audio_msg(false, 1000); + HELPER_ASSERT_SUCCESS(controller.write_audio(&audio_msg, &fmt)); + + controller.dispose(); +} diff --git a/trunk/src/utest/srs_utest_fmp4.hpp b/trunk/src/utest/srs_utest_fmp4.hpp new file mode 100644 index 000000000..c19aa4fe9 --- /dev/null +++ b/trunk/src/utest/srs_utest_fmp4.hpp @@ -0,0 +1,15 @@ +// +// Copyright (c) 2013-2025 The SRS Authors +// +// SPDX-License-Identifier: MIT +// + +#ifndef SRS_UTEST_FMP4_HPP +#define SRS_UTEST_FMP4_HPP + +/* +#include +*/ +#include + +#endif diff --git a/trunk/src/utest/srs_utest_http.cpp b/trunk/src/utest/srs_utest_http.cpp index d895c0b95..9c2528bae 100644 --- a/trunk/src/utest/srs_utest_http.cpp +++ b/trunk/src/utest/srs_utest_http.cpp @@ -1569,6 +1569,50 @@ VOID TEST(ProtocolHTTPTest, VodStreamHandlers) HELPER_ASSERT_SUCCESS(h.serve_http(&w2, &r)); __MOCK_HTTP_EXPECT_STREQ(200, "livestream-13.ts?hls_ctx=123456", w2); } + + // fmp4 .m4s with hls_ctx test + if (true) { + SrsHttpMuxEntry e; + e.pattern = "/"; + + SrsVodStream h("/tmp"); + h.set_fs_factory(new MockFileReaderFactory("livestream-13.m4s")); + h.set_path_check(_mock_srs_path_always_exists); + h.entry = &e; + + MockResponseWriter w; + SrsHttpMessage r(NULL, NULL); + HELPER_ASSERT_SUCCESS(r.set_url("/index.m3u8?hls_ctx=123456", false)); + + HELPER_ASSERT_SUCCESS(h.serve_http(&w, &r)); + __MOCK_HTTP_EXPECT_STREQ4(200, "/index.m3u8?hls_ctx=123456\n", w); + + MockResponseWriter w2; + HELPER_ASSERT_SUCCESS(h.serve_http(&w2, &r)); + __MOCK_HTTP_EXPECT_STREQ(200, "livestream-13.m4s?hls_ctx=123456", w2); + } + + // fmp4 init.mp4 with hls_ctx test + if (true) { + SrsHttpMuxEntry e; + e.pattern = "/"; + + SrsVodStream h("/tmp"); + h.set_fs_factory(new MockFileReaderFactory("init.mp4")); + h.set_path_check(_mock_srs_path_always_exists); + h.entry = &e; + + MockResponseWriter w; + SrsHttpMessage r(NULL, NULL); + HELPER_ASSERT_SUCCESS(r.set_url("/index.m3u8?hls_ctx=123456", false)); + + HELPER_ASSERT_SUCCESS(h.serve_http(&w, &r)); + __MOCK_HTTP_EXPECT_STREQ4(200, "/index.m3u8?hls_ctx=123456\n", w); + + MockResponseWriter w2; + HELPER_ASSERT_SUCCESS(h.serve_http(&w2, &r)); + __MOCK_HTTP_EXPECT_STREQ(200, "init.mp4?hls_ctx=123456", w2); + } } VOID TEST(ProtocolHTTPTest, BasicHandlers) diff --git a/trunk/src/utest/srs_utest_mp4.cpp b/trunk/src/utest/srs_utest_mp4.cpp index c4cb8c8f7..295ae1837 100644 --- a/trunk/src/utest/srs_utest_mp4.cpp +++ b/trunk/src/utest/srs_utest_mp4.cpp @@ -12,6 +12,7 @@ using namespace std; #include #include #include +#include VOID TEST(KernelMp4Test, PrintPadding) { @@ -535,6 +536,21 @@ VOID TEST(KernelMp4Test, FullBoxDump) } } +VOID TEST(KernelMp4Test, MOOFBox) +{ + if (true) { + SrsMp4MovieFragmentBox box; + + SrsMp4MovieFragmentHeaderBox *mfhd = new SrsMp4MovieFragmentHeaderBox(); + box.set_mfhd(mfhd); + EXPECT_EQ(box.mfhd(), mfhd); + + SrsMp4TrackFragmentBox* traf = new SrsMp4TrackFragmentBox(); + box.add_traf(traf); + EXPECT_TRUE(traf == box.get(SrsMp4BoxTypeTRAF)); + } +} + VOID TEST(KernelMp4Test, MFHDBox) { srs_error_t err; @@ -898,11 +914,10 @@ VOID TEST(KernelMp4Test, TREXBox) } SrsMp4MovieExtendsBox box; - EXPECT_TRUE(NULL == box.trex()); SrsMp4TrackExtendsBox* trex = new SrsMp4TrackExtendsBox(); - box.set_trex(trex); - EXPECT_TRUE(trex == box.trex()); + box.add_trex(trex); + EXPECT_TRUE(trex == box.get(SrsMp4BoxTypeTREX)); } VOID TEST(KernelMp4Test, TKHDBox) @@ -1823,6 +1838,442 @@ VOID TEST(KernelMp4Test, STSDBox) } } +VOID TEST(KernelMp4Test, SAIZBox) +{ + srs_error_t err; + // flags & 1 == 0; default_sample_info_size == 1 + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + uint8_t data[12+5]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIZ); b.write_1bytes(0); b.write_3bytes(0); + b.write_1bytes(1); b.write_4bytes(0); b.skip(-17); + + HELPER_ASSERT_SUCCESS(saiz.decode(&b)); + EXPECT_EQ(17, (int)saiz.nb_header()); + EXPECT_EQ(0, (int)saiz.version); + EXPECT_EQ(0, (int)saiz.flags); + EXPECT_EQ(1, (int)saiz.default_sample_info_size); + EXPECT_EQ(0, (int)saiz.sample_count); + EXPECT_EQ(0, saiz.sample_info_sizes.size()); + } + + // flags & 1 == 1; default_sample_info_size == 1 + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + uint8_t data[12+13]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIZ); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_1bytes(1); b.write_4bytes(0); b.skip(-25); + + HELPER_ASSERT_SUCCESS(saiz.decode(&b)); + EXPECT_EQ(25, (int)saiz.nb_header()); + EXPECT_EQ(0, (int)saiz.version); + EXPECT_EQ(1, (int)saiz.flags); + EXPECT_EQ(1, (int)saiz.aux_info_type); + EXPECT_EQ(2, (int)saiz.aux_info_type_parameter); + EXPECT_EQ(1, (int)saiz.default_sample_info_size); + EXPECT_EQ(0, (int)saiz.sample_count); + EXPECT_EQ(0, saiz.sample_info_sizes.size()); + } + + // flags & 1 == 1; default_sample_info_size == 0; sample_count = 3; + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + + uint8_t data[12+16]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIZ); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_1bytes(0); b.write_4bytes(3); + b.write_1bytes(4); b.write_1bytes(5); b.write_1bytes(6); + b.skip(-28); + + HELPER_ASSERT_SUCCESS(saiz.decode(&b)); + EXPECT_EQ(28, (int)saiz.nb_header()); + EXPECT_EQ(0, (int)saiz.version); + EXPECT_EQ(1, (int)saiz.flags); + EXPECT_EQ(1, (int)saiz.aux_info_type); + EXPECT_EQ(2, (int)saiz.aux_info_type_parameter); + EXPECT_EQ(0, (int)saiz.default_sample_info_size); + EXPECT_EQ(3, (int)saiz.sample_count); + EXPECT_EQ(3, saiz.sample_info_sizes.size()); + EXPECT_EQ(4, saiz.sample_info_sizes[0]); + EXPECT_EQ(5, saiz.sample_info_sizes[1]); + EXPECT_EQ(6, saiz.sample_info_sizes[2]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + saiz.flags = 0; + saiz.default_sample_info_size = 1; + saiz.sample_count = 0; + + EXPECT_EQ(17, saiz.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("default_sample_info_size=1, sample_count=0", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + saiz.flags = 1; + saiz.default_sample_info_size = 1; + saiz.sample_count = 0; + + EXPECT_EQ(25, saiz.nb_header()); + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("default_sample_info_size=1, sample_count=0", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoSizeBox saiz; + saiz.flags = 1; + saiz.default_sample_info_size = 0; + saiz.sample_count = 1; + saiz.sample_info_sizes.push_back(4); + + EXPECT_EQ(26, saiz.nb_header()); + stringstream ss; + SrsMp4DumpContext dc; + saiz.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("default_sample_info_size=0, sample_count=1", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SAIOBox) +{ + srs_error_t err; + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + uint8_t data[12+8]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIO); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(1); b.write_4bytes(2); b.skip(-20); + + HELPER_ASSERT_SUCCESS(saio.decode(&b)); + EXPECT_EQ(20, (int)saio.nb_header()); + EXPECT_EQ(0, (int)saio.version); + EXPECT_EQ(0, (int)saio.flags); + EXPECT_EQ(1, (int)saio.offsets.size()); + EXPECT_EQ(2, (int)saio.offsets[0]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + uint8_t data[12+16]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIO); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_4bytes(1); b.write_4bytes(2); b.skip(-28); + + HELPER_ASSERT_SUCCESS(saio.decode(&b)); + EXPECT_EQ(28, (int)saio.nb_header()); + EXPECT_EQ(0, (int)saio.version); + EXPECT_EQ(1, (int)saio.flags); + EXPECT_EQ(1, (int)saio.aux_info_type); + EXPECT_EQ(2, (int)saio.aux_info_type_parameter); + EXPECT_EQ(1, (int)saio.offsets.size()); + EXPECT_EQ(2, (int)saio.offsets[0]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + + uint8_t data[12+20]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSAIO); b.write_1bytes(1); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_4bytes(1); b.write_8bytes(2); b.skip(-32); + + HELPER_ASSERT_SUCCESS(saio.decode(&b)); + EXPECT_EQ(32, (int)saio.nb_header()); + EXPECT_EQ(1, (int)saio.version); + EXPECT_EQ(1, (int)saio.flags); + EXPECT_EQ(1, (int)saio.aux_info_type); + EXPECT_EQ(2, (int)saio.aux_info_type_parameter); + EXPECT_EQ(1, (int)saio.offsets.size()); + EXPECT_EQ(2, (int)saio.offsets[0]); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + saio.version = 0; + saio.flags = 0; + saio.offsets.push_back(2); + EXPECT_EQ(20, (int)saio.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("entry_count=1", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + saio.version = 0; + saio.flags = 1; + saio.offsets.push_back(2); + EXPECT_EQ(28, (int)saio.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("entry_count=1", v.c_str()); + } + + if (true) { + SrsMp4SampleAuxiliaryInfoOffsetBox saio; + saio.version = 1; + saio.flags = 1; + saio.offsets.push_back(2); + EXPECT_EQ(32, (int)saio.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + saio.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("entry_count=1", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SENCBox) +{ + srs_error_t err; + + if (true) { + SrsMp4SampleEncryptionBox senc(8); + + uint8_t data[12+4]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSENC); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(0); b.skip(-16); + + HELPER_ASSERT_SUCCESS(senc.decode(&b)); + EXPECT_EQ(16, (int)senc.nb_header()); + EXPECT_EQ(0, (int)senc.version); + EXPECT_EQ(0, (int)senc.flags); + EXPECT_EQ(0, (int)senc.entries.size()); + } + + if (true) { + SrsMp4SampleEncryptionBox senc(8); + + uint8_t data[12+12]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSENC); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(1); b.write_8bytes(1); b.skip(-24); + + HELPER_ASSERT_SUCCESS(senc.decode(&b)); + EXPECT_EQ(24, (int)senc.nb_header()); + EXPECT_EQ(0, (int)senc.version); + EXPECT_EQ(0, (int)senc.flags); + EXPECT_EQ(1, (int)senc.entries.size()); + } +} + +VOID TEST(KernelMp4Test, FRMABox) +{ + srs_error_t err; + + if (true) { + SrsMp4OriginalFormatBox frma(1); + uint8_t data[8+4]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(0); b.write_4bytes(SrsMp4BoxTypeFRMA); + b.write_4bytes(1); b.skip(-12); + + HELPER_ASSERT_SUCCESS(frma.decode(&b)); + EXPECT_EQ(12, (int)frma.nb_header()); + } + + if (true) { + SrsMp4OriginalFormatBox frma(1); + EXPECT_EQ(12, (int)frma.nb_header()); + + stringstream ss; + SrsMp4DumpContext dc; + frma.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("original format=1\n", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SCHMBox) { + srs_error_t err; + + if (true) { + SrsMp4SchemeTypeBox schm; + uint8_t data[12+8]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSCHM); b.write_1bytes(0); b.write_3bytes(0); + b.write_4bytes(1); b.write_4bytes(2); b.skip(-20); + + HELPER_ASSERT_SUCCESS(schm.decode(&b)); + EXPECT_EQ(20, (int)schm.nb_header()); + EXPECT_EQ(0, (int)schm.version); + EXPECT_EQ(0, (int)schm.flags); + EXPECT_EQ(1, (int)schm.scheme_type); + EXPECT_EQ(2, (int)schm.scheme_version); + } + + if (true) { + SrsMp4SchemeTypeBox schm; + uint8_t data[12+8+4]; + SrsBuffer b((char*)data, sizeof(data)); + b.write_4bytes(16); b.write_4bytes(SrsMp4BoxTypeSCHM); b.write_1bytes(0); b.write_3bytes(1); + b.write_4bytes(1); b.write_4bytes(2); + b.write_1bytes(65); b.write_1bytes(65); b.write_1bytes(65); b.write_1bytes(0); b.skip(-24); + + HELPER_ASSERT_SUCCESS(schm.decode(&b)); + EXPECT_EQ(24, (int)schm.nb_header()); + EXPECT_EQ(0, (int)schm.version); + EXPECT_EQ(1, (int)schm.flags); + EXPECT_EQ(1, (int)schm.scheme_type); + EXPECT_EQ(2, (int)schm.scheme_version); + + stringstream ss; + SrsMp4DumpContext dc; + schm.dumps_detail(ss, dc); + string v = ss.str(); + EXPECT_STREQ("scheme_type=1, scheme_version=2\nscheme_uri=AAA\n", v.c_str()); + } +} + +VOID TEST(KernelMp4Test, SrsInitMp4Segment) +{ + srs_error_t err; + // write single video segment + if (true) { + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init.mp4"); + SrsFormat fmt; + HELPER_ASSERT_SUCCESS(fmt.initialize()); + + uint8_t raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + HELPER_ASSERT_SUCCESS(fmt.on_video(0, (char*)raw, sizeof(raw))); + + HELPER_ASSERT_SUCCESS(segment.write_video_only(&fmt, 1)); + EXPECT_TRUE(fw.filesize() > 0); + } + + // write single audio segment + if (true) { + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init.mp4"); + SrsFormat fmt; + HELPER_ASSERT_SUCCESS(fmt.initialize()); + + uint8_t raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + + HELPER_ASSERT_SUCCESS(fmt.on_audio(0, (char*)raw, sizeof(raw))); + + HELPER_ASSERT_SUCCESS(segment.write_audio_only(&fmt, 1)); + EXPECT_TRUE(fw.filesize() > 0); + } + + // write both audio and video segment + if (true) { + MockSrsFileWriter fw; + SrsInitMp4Segment segment(&fw); + + segment.set_path("/tmp/init.mp4"); + SrsFormat fmt; + HELPER_ASSERT_SUCCESS(fmt.initialize()); + + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + + HELPER_ASSERT_SUCCESS(fmt.on_video(0, (char*)video_raw, sizeof(video_raw))); + + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + + HELPER_ASSERT_SUCCESS(fmt.on_audio(0, (char*)audio_raw, sizeof(audio_raw))); + + HELPER_ASSERT_SUCCESS(segment.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); + } +} + +VOID TEST(KernelMp4Test, SrsFmp4SegmentEncoder) +{ + srs_error_t err; + + if (true) { + MockSrsFileWriter fw; + SrsFmp4SegmentEncoder encoder; + encoder.initialize(&fw, 0, 0, 1, 2); + + SrsFormat video_fmt; + HELPER_ASSERT_SUCCESS(video_fmt.initialize()); + + SrsFormat audio_fmt; + HELPER_ASSERT_SUCCESS(audio_fmt.initialize()); + + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + + HELPER_ASSERT_SUCCESS(video_fmt.on_video(0, (char*)video_raw, sizeof(video_raw))); + + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + + HELPER_ASSERT_SUCCESS(audio_fmt.on_audio(0, (char*)audio_raw, sizeof(audio_raw))); + + SrsVideoAvcFrameType video_frame_type = video_fmt.video->frame_type; + uint32_t cts = (uint32_t)video_fmt.video->cts; + + uint32_t dts = 0; + uint32_t pts = dts + cts; + + uint8_t* video_sample = (uint8_t*)video_fmt.raw; + uint32_t nb_video_sample = (uint32_t)video_fmt.nb_raw; + encoder.write_sample(SrsMp4HandlerTypeVIDE, video_frame_type, dts, pts, video_sample, nb_video_sample); + uint8_t* audio_sample = (uint8_t*)audio_fmt.raw; + uint32_t nb_audio_sample = (uint32_t)audio_fmt.nb_raw; + encoder.write_sample(SrsMp4HandlerTypeSOUN, 0, 0, 0, audio_sample, nb_audio_sample); + encoder.flush(dts); + EXPECT_TRUE(fw.filesize() > 0); + } +} + VOID TEST(KernelMp4Test, SrsMp4M2tsInitEncoder) { srs_error_t err; @@ -1867,5 +2318,32 @@ VOID TEST(KernelMp4Test, SrsMp4M2tsInitEncoder) HELPER_ASSERT_SUCCESS(enc.write(&fmt, false, 1)); EXPECT_TRUE(fw.filesize() > 0); } + + if (true) { + MockSrsFileWriter fw; + HELPER_ASSERT_SUCCESS(fw.open("test.mp4")); + + SrsMp4M2tsInitEncoder enc; + HELPER_ASSERT_SUCCESS(enc.initialize(&fw)); + + SrsFormat fmt; + EXPECT_TRUE(srs_success == fmt.initialize()); + + uint8_t video_raw[] = { + 0x17, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x64, 0x00, 0x20, 0xff, 0xe1, 0x00, 0x19, 0x67, 0x64, 0x00, 0x20, + 0xac, 0xd9, 0x40, 0xc0, 0x29, 0xb0, 0x11, 0x00, 0x00, 0x03, 0x00, 0x01, 0x00, 0x00, 0x03, 0x00, + 0x32, 0x0f, 0x18, 0x31, 0x96, 0x01, 0x00, 0x05, 0x68, 0xeb, 0xec, 0xb2, 0x2c + }; + EXPECT_TRUE(srs_success == fmt.on_video(0, (char*)video_raw, sizeof(video_raw))); + + uint8_t audio_raw[] = { + 0xaf, 0x00, 0x12, 0x10 + }; + EXPECT_TRUE(srs_success == fmt.on_audio(0, (char*)audio_raw, sizeof(audio_raw))); + + HELPER_ASSERT_SUCCESS(enc.write(&fmt, 1, 2)); + EXPECT_TRUE(fw.filesize() > 0); + } }