From 4246be92c99301a888ea6ecef67f881a2273265f Mon Sep 17 00:00:00 2001 From: winlin Date: Sat, 31 Jan 2015 21:16:42 +0800 Subject: [PATCH] for #250, merge avc to codec. use queue to dequeue. --- trunk/configure | 2 +- trunk/ide/srs_upp/srs_upp.upp | 2 - trunk/src/app/srs_app_hls.cpp | 2 +- trunk/src/app/srs_app_mpegts_udp.cpp | 90 +++- trunk/src/app/srs_app_mpegts_udp.hpp | 23 + trunk/src/app/srs_app_source.cpp | 2 +- trunk/src/kernel/srs_kernel_avc.cpp | 612 -------------------------- trunk/src/kernel/srs_kernel_avc.hpp | 304 ------------- trunk/src/kernel/srs_kernel_codec.cpp | 584 ++++++++++++++++++++++++ trunk/src/kernel/srs_kernel_codec.hpp | 263 +++++++++++ trunk/src/kernel/srs_kernel_ts.cpp | 2 +- trunk/src/libs/srs_librtmp.cpp | 1 - 12 files changed, 959 insertions(+), 928 deletions(-) delete mode 100644 trunk/src/kernel/srs_kernel_avc.cpp delete mode 100644 trunk/src/kernel/srs_kernel_avc.hpp diff --git a/trunk/configure b/trunk/configure index c7635ecc5..1a1ecf677 100755 --- a/trunk/configure +++ b/trunk/configure @@ -366,7 +366,7 @@ ModuleLibIncs=(${SRS_OBJS_DIR}) MODULE_FILES=("srs_kernel_error" "srs_kernel_log" "srs_kernel_stream" "srs_kernel_utility" "srs_kernel_flv" "srs_kernel_codec" "srs_kernel_file" "srs_kernel_consts" "srs_kernel_aac" "srs_kernel_mp3" "srs_kernel_ts" - "srs_kernel_avc" "srs_kernel_buffer") + "srs_kernel_buffer") KERNEL_INCS="src/kernel"; MODULE_DIR=${KERNEL_INCS} . auto/modules.sh KERNEL_OBJS="${MODULE_OBJS[@]}" # diff --git a/trunk/ide/srs_upp/srs_upp.upp b/trunk/ide/srs_upp/srs_upp.upp index 138cd3081..7749a6949 100755 --- a/trunk/ide/srs_upp/srs_upp.upp +++ b/trunk/ide/srs_upp/srs_upp.upp @@ -20,8 +20,6 @@ file kernel readonly separator, ../../src/kernel/srs_kernel_aac.hpp, ../../src/kernel/srs_kernel_aac.cpp, - ../../src/kernel/srs_kernel_avc.hpp, - ../../src/kernel/srs_kernel_avc.cpp, ../../src/kernel/srs_kernel_buffer.hpp, ../../src/kernel/srs_kernel_buffer.cpp, ../../src/kernel/srs_kernel_codec.hpp, diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index a9da852e2..8e749142f 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -47,7 +47,7 @@ using namespace std; #include #include #include -#include +#include #include #include #include diff --git a/trunk/src/app/srs_app_mpegts_udp.cpp b/trunk/src/app/srs_app_mpegts_udp.cpp index 0738aa948..aa86cb0d7 100644 --- a/trunk/src/app/srs_app_mpegts_udp.cpp +++ b/trunk/src/app/srs_app_mpegts_udp.cpp @@ -57,6 +57,68 @@ ISrsUdpHandler::~ISrsUdpHandler() { } +SrsMpegtsQueue::SrsMpegtsQueue() +{ + nb_audios = nb_videos = 0; +} + +SrsMpegtsQueue::~SrsMpegtsQueue() +{ + std::map::iterator it; + for (it = msgs.begin(); it != msgs.end(); ++it) { + SrsSharedPtrMessage* msg = it->second; + srs_freep(msg); + } + msgs.clear(); +} + +int SrsMpegtsQueue::push(SrsSharedPtrMessage* msg) +{ + int ret = ERROR_SUCCESS; + + if (msgs.find(msg->timestamp) != msgs.end()) { + srs_warn("mpegts: free the msg for dts exists, dts=%"PRId64, msg->timestamp); + srs_freep(msg); + return ret; + } + + if (msg->is_audio()) { + nb_audios++; + } + + if (msg->is_video()) { + nb_videos++; + } + + msgs[msg->timestamp] = msg; + + return ret; +} + +SrsSharedPtrMessage* SrsMpegtsQueue::dequeue() +{ + // got 2+ videos and audios, ok to dequeue. + bool av_ok = nb_videos >= 2 && nb_audios >= 2; + // 100 videos about 30s, while 300 audios about 30s + bool av_overflow = nb_videos > 100 || nb_audios > 300; + + if (av_ok || av_overflow) { + std::map::iterator it = msgs.begin(); + SrsSharedPtrMessage* msg = it->second; + msgs.erase(it); + + if (msg->is_audio()) { + nb_audios--; + } + + if (msg->is_video()) { + nb_videos--; + } + } + + return NULL; +} + SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c) { stream = new SrsStream(); @@ -72,6 +134,7 @@ SrsMpegtsOverUdp::SrsMpegtsOverUdp(SrsConfDirective* c) h264_sps_changed = false; h264_pps_changed = false; h264_sps_pps_sent = false; + queue = new SrsMpegtsQueue(); } SrsMpegtsOverUdp::~SrsMpegtsOverUdp() @@ -82,6 +145,7 @@ SrsMpegtsOverUdp::~SrsMpegtsOverUdp() srs_freep(stream); srs_freep(context); srs_freep(avc); + srs_freep(queue); } int SrsMpegtsOverUdp::on_udp_packet(sockaddr_in* from, char* buf, int nb_buf) @@ -280,11 +344,14 @@ int SrsMpegtsOverUdp::on_ts_video(SrsTsMessage* msg, SrsStream* avs) // it may be return error, but we must process all packets. if ((ret = write_h264_raw_frame(frame, frame_size, dts, pts)) != ERROR_SUCCESS) { - if (ret = ERROR_H264_DROP_BEFORE_SPS_PPS) { + if (ret == ERROR_H264_DROP_BEFORE_SPS_PPS) { continue; } return ret; } + + // for video, drop others with same pts/dts. + break; } return ret; @@ -399,14 +466,27 @@ int SrsMpegtsOverUdp::rtmp_write_packet(char type, u_int32_t timestamp, char* da SrsSharedPtrMessage* msg = NULL; if ((ret = srs_rtmp_create_msg(type, timestamp, data, size, stream_id, &msg)) != ERROR_SUCCESS) { + srs_error("mpegts: create shared ptr msg failed. ret=%d", ret); + return ret; + } + srs_assert(msg); + + // push msg to queue. + if ((ret = queue->push(msg)) != ERROR_SUCCESS) { + srs_error("mpegts: push msg to queue failed. ret=%d", ret); return ret; } - srs_assert(msg); + // for all ready msg, dequeue and send out. + for (;;) { + if ((msg = queue->dequeue()) == NULL) { + break; + } - // send out encoded msg. - if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { - return ret; + // send out encoded msg. + if ((ret = client->send_and_free_message(msg, stream_id)) != ERROR_SUCCESS) { + return ret; + } } return ret; diff --git a/trunk/src/app/srs_app_mpegts_udp.hpp b/trunk/src/app/srs_app_mpegts_udp.hpp index f51860e86..323732722 100644 --- a/trunk/src/app/srs_app_mpegts_udp.hpp +++ b/trunk/src/app/srs_app_mpegts_udp.hpp @@ -34,6 +34,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. class sockaddr_in; #include +#include class SrsStream; class SrsTsContext; @@ -43,6 +44,7 @@ class SrsRtmpClient; class SrsStSocket; class SrsRequest; class SrsRawH264Stream; +class SrsSharedPtrMessage; #include #include @@ -68,6 +70,26 @@ public: virtual int on_udp_packet(sockaddr_in* from, char* buf, int nb_buf) = 0; }; +/** +* the queue for mpegts over udp to send packets. +* for the aac in mpegts contains many flv packets in a pes packet, +* we must recalc the timestamp. +*/ +class SrsMpegtsQueue +{ +private: + // key: dts, value: msg. + std::map msgs; + int nb_audios; + int nb_videos; +public: + SrsMpegtsQueue(); + virtual ~SrsMpegtsQueue(); +public: + virtual int push(SrsSharedPtrMessage* msg); + virtual SrsSharedPtrMessage* dequeue(); +}; + /** * the mpegts over udp stream caster. */ @@ -92,6 +114,7 @@ private: std::string h264_pps; bool h264_pps_changed; bool h264_sps_pps_sent; + SrsMpegtsQueue* queue; public: SrsMpegtsOverUdp(SrsConfDirective* c); virtual ~SrsMpegtsOverUdp(); diff --git a/trunk/src/app/srs_app_source.cpp b/trunk/src/app/srs_app_source.cpp index 10a8bfb33..fa88d5b18 100644 --- a/trunk/src/app/srs_app_source.cpp +++ b/trunk/src/app/srs_app_source.cpp @@ -40,7 +40,7 @@ using namespace std; #include #include #include -#include +#include #include #define CONST_MAX_JITTER_MS 500 diff --git a/trunk/src/kernel/srs_kernel_avc.cpp b/trunk/src/kernel/srs_kernel_avc.cpp deleted file mode 100644 index 8dc9112a3..000000000 --- a/trunk/src/kernel/srs_kernel_avc.cpp +++ /dev/null @@ -1,612 +0,0 @@ -/* -The MIT License (MIT) - -Copyright (c) 2013-2015 winlin - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#include - -#include -#include -#include -#include -#include -#include - -using namespace std; - -SrsCodecSampleUnit::SrsCodecSampleUnit() -{ - size = 0; - bytes = NULL; -} - -SrsCodecSampleUnit::~SrsCodecSampleUnit() -{ -} - -SrsCodecSample::SrsCodecSample() -{ - clear(); -} - -SrsCodecSample::~SrsCodecSample() -{ -} - -void SrsCodecSample::clear() -{ - is_video = false; - nb_sample_units = 0; - - cts = 0; - frame_type = SrsCodecVideoAVCFrameReserved; - avc_packet_type = SrsCodecVideoAVCTypeReserved; - - acodec = SrsCodecAudioReserved1; - sound_rate = SrsCodecAudioSampleRateReserved; - sound_size = SrsCodecAudioSampleSizeReserved; - sound_type = SrsCodecAudioSoundTypeReserved; - aac_packet_type = SrsCodecAudioTypeReserved; -} - -int SrsCodecSample::add_sample_unit(char* bytes, int size) -{ - int ret = ERROR_SUCCESS; - - if (nb_sample_units >= __SRS_SRS_MAX_CODEC_SAMPLE) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("hls decode samples error, " - "exceed the max count: %d, ret=%d", __SRS_SRS_MAX_CODEC_SAMPLE, ret); - return ret; - } - - SrsCodecSampleUnit* sample_unit = &sample_units[nb_sample_units++]; - sample_unit->bytes = bytes; - sample_unit->size = size; - - return ret; -} - -SrsAvcAacCodec::SrsAvcAacCodec() -{ - width = 0; - height = 0; - duration = 0; - NAL_unit_length = 0; - frame_rate = 0; - - video_data_rate = 0; - video_codec_id = 0; - - audio_data_rate = 0; - audio_codec_id = 0; - - avc_profile = 0; - avc_level = 0; - aac_profile = 0; - aac_sample_rate = __SRS_AAC_SAMPLE_RATE_UNSET; // sample rate ignored - aac_channels = 0; - avc_extra_size = 0; - avc_extra_data = NULL; - aac_extra_size = 0; - aac_extra_data = NULL; - - sequenceParameterSetLength = 0; - sequenceParameterSetNALUnit = NULL; - pictureParameterSetLength = 0; - pictureParameterSetNALUnit = NULL; - - stream = new SrsStream(); -} - -SrsAvcAacCodec::~SrsAvcAacCodec() -{ - srs_freep(avc_extra_data); - srs_freep(aac_extra_data); - - srs_freep(stream); - srs_freep(sequenceParameterSetNALUnit); - srs_freep(pictureParameterSetNALUnit); -} - -int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample) -{ - int ret = ERROR_SUCCESS; - - sample->is_video = false; - - if (!data || size <= 0) { - srs_trace("no audio present, ignore it."); - return ret; - } - - if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) { - return ret; - } - - // audio decode - if (!stream->require(1)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("audio codec decode sound_format failed. ret=%d", ret); - return ret; - } - - // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76 - int8_t sound_format = stream->read_1bytes(); - - int8_t sound_type = sound_format & 0x01; - int8_t sound_size = (sound_format >> 1) & 0x01; - int8_t sound_rate = (sound_format >> 2) & 0x03; - sound_format = (sound_format >> 4) & 0x0f; - - audio_codec_id = sound_format; - sample->acodec = (SrsCodecAudio)audio_codec_id; - - sample->sound_type = (SrsCodecAudioSoundType)sound_type; - sample->sound_rate = (SrsCodecAudioSampleRate)sound_rate; - sample->sound_size = (SrsCodecAudioSampleSize)sound_size; - - // we support h.264+mp3 for hls. - if (audio_codec_id == SrsCodecAudioMP3) { - return ERROR_HLS_TRY_MP3; - } - - // only support aac - if (audio_codec_id != SrsCodecAudioAAC) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("audio codec only support mp3/aac codec. actual=%d, ret=%d", audio_codec_id, ret); - return ret; - } - - if (!stream->require(1)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("audio codec decode aac_packet_type failed. ret=%d", ret); - return ret; - } - - int8_t aac_packet_type = stream->read_1bytes(); - sample->aac_packet_type = (SrsCodecAudioType)aac_packet_type; - - if (aac_packet_type == SrsCodecAudioTypeSequenceHeader) { - // AudioSpecificConfig - // 1.6.2.1 AudioSpecificConfig, in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf, page 33. - aac_extra_size = stream->size() - stream->pos(); - if (aac_extra_size > 0) { - srs_freep(aac_extra_data); - aac_extra_data = new char[aac_extra_size]; - memcpy(aac_extra_data, stream->data() + stream->pos(), aac_extra_size); - } - - // only need to decode the first 2bytes: - // audioObjectType, aac_profile, 5bits. - // samplingFrequencyIndex, aac_sample_rate, 4bits. - // channelConfiguration, aac_channels, 4bits - if (!stream->require(2)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("audio codec decode aac sequence header failed. ret=%d", ret); - return ret; - } - aac_profile = stream->read_1bytes(); - aac_sample_rate = stream->read_1bytes(); - - aac_channels = (aac_sample_rate >> 3) & 0x0f; - aac_sample_rate = ((aac_profile << 1) & 0x0e) | ((aac_sample_rate >> 7) & 0x01); - aac_profile = (aac_profile >> 3) & 0x1f; - - if (aac_profile == 0 || aac_profile == 0x1f) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("audio codec decode aac sequence header failed, " - "adts object=%d invalid. ret=%d", aac_profile, ret); - return ret; - } - - // TODO: FIXME: to support aac he/he-v2, see: ngx_rtmp_codec_parse_aac_header - // @see: https://github.com/winlinvip/nginx-rtmp-module/commit/3a5f9eea78fc8d11e8be922aea9ac349b9dcbfc2 - // - // donot force to LC, @see: https://github.com/winlinvip/simple-rtmp-server/issues/81 - // the source will print the sequence header info. - //if (aac_profile > 3) { - // Mark all extended profiles as LC - // to make Android as happy as possible. - // @see: ngx_rtmp_hls_parse_aac_header - //aac_profile = 1; - //} - } else if (aac_packet_type == SrsCodecAudioTypeRawData) { - // ensure the sequence header demuxed - if (aac_extra_size <= 0 || !aac_extra_data) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("audio codec decode aac failed, sequence header not found. ret=%d", ret); - return ret; - } - - // Raw AAC frame data in UI8 [] - // 6.3 Raw Data, aac-iso-13818-7.pdf, page 28 - if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), stream->size() - stream->pos())) != ERROR_SUCCESS) { - srs_error("audio codec add sample failed. ret=%d", ret); - return ret; - } - } else { - // ignored. - } - - // reset the sample rate by sequence header - if (aac_sample_rate != __SRS_AAC_SAMPLE_RATE_UNSET) { - static int aac_sample_rates[] = { - 96000, 88200, 64000, 48000, - 44100, 32000, 24000, 22050, - 16000, 12000, 11025, 8000, - 7350, 0, 0, 0 - }; - switch (aac_sample_rates[aac_sample_rate]) { - case 11025: - sample->sound_rate = SrsCodecAudioSampleRate11025; - break; - case 22050: - sample->sound_rate = SrsCodecAudioSampleRate22050; - break; - case 44100: - sample->sound_rate = SrsCodecAudioSampleRate44100; - break; - default: - break; - }; - } - - srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", - sound_type, audio_codec_id, sound_size, sound_rate, sound_format, size); - - return ret; -} - -int SrsAvcAacCodec::audio_mp3_demux(char* data, int size, SrsCodecSample* sample) -{ - int ret = ERROR_SUCCESS; - - // we always decode aac then mp3. - srs_assert(sample->acodec == SrsCodecAudioMP3); - - // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76 - if (!data || size <= 1) { - srs_trace("no mp3 audio present, ignore it."); - return ret; - } - - // mp3 payload. - if ((ret = sample->add_sample_unit(data + 1, size - 1)) != ERROR_SUCCESS) { - srs_error("audio codec add mp3 sample failed. ret=%d", ret); - return ret; - } - - srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", - sample->sound_type, audio_codec_id, sample->sound_size, sample->sound_rate, sample->acodec, size); - - return ret; -} - -int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample) -{ - int ret = ERROR_SUCCESS; - - sample->is_video = true; - - if (!data || size <= 0) { - srs_trace("no video present, ignore it."); - return ret; - } - - if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) { - return ret; - } - - // video decode - if (!stream->require(1)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("video codec decode frame_type failed. ret=%d", ret); - return ret; - } - - // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 - int8_t frame_type = stream->read_1bytes(); - int8_t codec_id = frame_type & 0x0f; - frame_type = (frame_type >> 4) & 0x0f; - - sample->frame_type = (SrsCodecVideoAVCFrame)frame_type; - - // ignore info frame without error, - // @see https://github.com/winlinvip/simple-rtmp-server/issues/288#issuecomment-69863909 - if (sample->frame_type == SrsCodecVideoAVCFrameVideoInfoFrame) { - srs_warn("video codec igone the info frame, ret=%d", ret); - return ret; - } - - // only support h.264/avc - if (codec_id != SrsCodecVideoAVC) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("video codec only support video h.264/avc codec. actual=%d, ret=%d", codec_id, ret); - return ret; - } - video_codec_id = codec_id; - - if (!stream->require(4)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("video codec decode avc_packet_type failed. ret=%d", ret); - return ret; - } - int8_t avc_packet_type = stream->read_1bytes(); - int32_t composition_time = stream->read_3bytes(); - - // pts = dts + cts. - sample->cts = composition_time; - sample->avc_packet_type = (SrsCodecVideoAVCType)avc_packet_type; - - if (avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) { - if ((ret = avc_demux_sps_pps(stream)) != ERROR_SUCCESS) { - return ret; - } - } else if (avc_packet_type == SrsCodecVideoAVCTypeNALU){ - // ensure the sequence header demuxed - if (avc_extra_size <= 0 || !avc_extra_data) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode failed, sequence header not found. ret=%d", ret); - return ret; - } - - // One or more NALUs (Full frames are required) - // try "AnnexB" from H.264-AVC-ISO_IEC_14496-10.pdf, page 211. - if ((ret = avc_demux_annexb_format(stream, sample)) != ERROR_SUCCESS) { - // stop try when system error. - if (ret != ERROR_HLS_AVC_TRY_OTHERS) { - srs_error("avc demux for annexb failed. ret=%d", ret); - return ret; - } - - // try "ISO Base Media File Format" from H.264-AVC-ISO_IEC_14496-15.pdf, page 20 - if ((ret = avc_demux_ibmf_format(stream, sample)) != ERROR_SUCCESS) { - return ret; - } - } - } else { - // ignored. - } - - srs_info("video decoded, type=%d, codec=%d, avc=%d, time=%d, size=%d", - frame_type, video_codec_id, avc_packet_type, composition_time, size); - - return ret; -} - -int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) -{ - int ret = ERROR_SUCCESS; - - // AVCDecoderConfigurationRecord - // 5.2.4.1.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 - avc_extra_size = stream->size() - stream->pos(); - if (avc_extra_size > 0) { - srs_freep(avc_extra_data); - avc_extra_data = new char[avc_extra_size]; - memcpy(avc_extra_data, stream->data() + stream->pos(), avc_extra_size); - } - - if (!stream->require(6)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header failed. ret=%d", ret); - return ret; - } - //int8_t configurationVersion = stream->read_1bytes(); - stream->read_1bytes(); - //int8_t AVCProfileIndication = stream->read_1bytes(); - avc_profile = stream->read_1bytes(); - //int8_t profile_compatibility = stream->read_1bytes(); - stream->read_1bytes(); - //int8_t AVCLevelIndication = stream->read_1bytes(); - avc_level = stream->read_1bytes(); - - // parse the NALU size. - int8_t lengthSizeMinusOne = stream->read_1bytes(); - lengthSizeMinusOne &= 0x03; - NAL_unit_length = lengthSizeMinusOne; - - // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 - // 5.2.4.1 AVC decoder configuration record - // 5.2.4.1.2 Semantics - // The value of this field shall be one of 0, 1, or 3 corresponding to a - // length encoded with 1, 2, or 4 bytes, respectively. - if (NAL_unit_length == 2) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("sps lengthSizeMinusOne should never be 2. ret=%d", ret); - return ret; - } - - // 1 sps - if (!stream->require(1)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header sps failed. ret=%d", ret); - return ret; - } - int8_t numOfSequenceParameterSets = stream->read_1bytes(); - numOfSequenceParameterSets &= 0x1f; - if (numOfSequenceParameterSets != 1) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header sps failed. ret=%d", ret); - return ret; - } - if (!stream->require(2)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header sps size failed. ret=%d", ret); - return ret; - } - sequenceParameterSetLength = stream->read_2bytes(); - if (!stream->require(sequenceParameterSetLength)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header sps data failed. ret=%d", ret); - return ret; - } - if (sequenceParameterSetLength > 0) { - srs_freep(sequenceParameterSetNALUnit); - sequenceParameterSetNALUnit = new char[sequenceParameterSetLength]; - memcpy(sequenceParameterSetNALUnit, stream->data() + stream->pos(), sequenceParameterSetLength); - stream->skip(sequenceParameterSetLength); - } - // 1 pps - if (!stream->require(1)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header pps failed. ret=%d", ret); - return ret; - } - int8_t numOfPictureParameterSets = stream->read_1bytes(); - numOfPictureParameterSets &= 0x1f; - if (numOfPictureParameterSets != 1) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header pps failed. ret=%d", ret); - return ret; - } - if (!stream->require(2)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header pps size failed. ret=%d", ret); - return ret; - } - pictureParameterSetLength = stream->read_2bytes(); - if (!stream->require(pictureParameterSetLength)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode sequenc header pps data failed. ret=%d", ret); - return ret; - } - if (pictureParameterSetLength > 0) { - srs_freep(pictureParameterSetNALUnit); - pictureParameterSetNALUnit = new char[pictureParameterSetLength]; - memcpy(pictureParameterSetNALUnit, stream->data() + stream->pos(), pictureParameterSetLength); - stream->skip(pictureParameterSetLength); - } - - return ret; -} - -int SrsAvcAacCodec::avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample) -{ - int ret = ERROR_SUCCESS; - - // not annexb, try others - if (!srs_avc_startswith_annexb(stream, NULL)) { - return ERROR_HLS_AVC_TRY_OTHERS; - } - - // AnnexB - // B.1.1 Byte stream NAL unit syntax, - // H.264-AVC-ISO_IEC_14496-10.pdf, page 211. - while (!stream->empty()) { - // find start code - int nb_start_code = 0; - if (!srs_avc_startswith_annexb(stream, &nb_start_code)) { - return ret; - } - - // skip the start code. - if (nb_start_code > 0) { - stream->skip(nb_start_code); - } - - // the NALU start bytes. - char* p = stream->data() + stream->pos(); - - // get the last matched NALU - while (!stream->empty()) { - if (srs_avc_startswith_annexb(stream, NULL)) { - break; - } - - stream->skip(1); - } - - char* pp = stream->data() + stream->pos(); - - // skip the empty. - if (pp - p <= 0) { - continue; - } - - // got the NALU. - if ((ret = sample->add_sample_unit(p, pp - p)) != ERROR_SUCCESS) { - srs_error("annexb add video sample failed. ret=%d", ret); - return ret; - } - } - - return ret; -} - -int SrsAvcAacCodec::avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample) -{ - int ret = ERROR_SUCCESS; - - int PictureLength = stream->size() - stream->pos(); - - // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 - // 5.2.4.1 AVC decoder configuration record - // 5.2.4.1.2 Semantics - // The value of this field shall be one of 0, 1, or 3 corresponding to a - // length encoded with 1, 2, or 4 bytes, respectively. - srs_assert(NAL_unit_length != 2); - - // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 20 - for (int i = 0; i < PictureLength;) { - // unsigned int((NAL_unit_length+1)*8) NALUnitLength; - if (!stream->require(NAL_unit_length + 1)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode NALU size failed. ret=%d", ret); - return ret; - } - int32_t NALUnitLength = 0; - if (NAL_unit_length == 3) { - NALUnitLength = stream->read_4bytes(); - } else if (NAL_unit_length == 1) { - NALUnitLength = stream->read_2bytes(); - } else { - NALUnitLength = stream->read_1bytes(); - } - - // maybe stream is invalid format. - // see: https://github.com/winlinvip/simple-rtmp-server/issues/183 - if (NALUnitLength < 0) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("maybe stream is AnnexB format. ret=%d", ret); - return ret; - } - - // NALUnit - if (!stream->require(NALUnitLength)) { - ret = ERROR_HLS_DECODE_ERROR; - srs_error("avc decode NALU data failed. ret=%d", ret); - return ret; - } - // 7.3.1 NAL unit syntax, H.264-AVC-ISO_IEC_14496-10.pdf, page 44. - if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), NALUnitLength)) != ERROR_SUCCESS) { - srs_error("avc add video sample failed. ret=%d", ret); - return ret; - } - stream->skip(NALUnitLength); - - i += NAL_unit_length + 1 + NALUnitLength; - } - - return ret; -} - diff --git a/trunk/src/kernel/srs_kernel_avc.hpp b/trunk/src/kernel/srs_kernel_avc.hpp deleted file mode 100644 index 126e865f2..000000000 --- a/trunk/src/kernel/srs_kernel_avc.hpp +++ /dev/null @@ -1,304 +0,0 @@ -/* -The MIT License (MIT) - -Copyright (c) 2013-2015 winlin - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -*/ - -#ifndef SRS_KERNEL_AVC_HPP -#define SRS_KERNEL_AVC_HPP - -/* -#include -*/ - -#include - -#include - -#include - -class SrsStream; -class SrsMpegtsFrame; -class SrsSimpleBuffer; -class SrsAvcAacCodec; -class SrsCodecSample; -class SrsFileWriter; - -/** -* the public data, event HLS disable, others can use it. -*/ -/** -* the flv sample rate map -*/ -extern int flv_sample_rates[]; - -/** -* the aac sample rate map -*/ -extern int aac_sample_rates[]; - -#define __SRS_SRS_MAX_CODEC_SAMPLE 128 -#define __SRS_AAC_SAMPLE_RATE_UNSET 15 - -// in ms, for HLS aac flush the audio -#define SRS_CONF_DEFAULT_AAC_DELAY 100 - -// max PES packets size to flush the video. -#define SRS_AUTO_HLS_AUDIO_CACHE_SIZE 1024 * 1024 - -/** -* the FLV/RTMP supported audio sample size. -* Size of each audio sample. This parameter only pertains to -* uncompressed formats. Compressed formats always decode -* to 16 bits internally. -* 0 = 8-bit samples -* 1 = 16-bit samples -*/ -enum SrsCodecAudioSampleSize -{ - // set to the max value to reserved, for array map. - SrsCodecAudioSampleSizeReserved = 2, - - SrsCodecAudioSampleSize8bit = 0, - SrsCodecAudioSampleSize16bit = 1, -}; - -/** -* the FLV/RTMP supported audio sound type/channel. -* Mono or stereo sound -* 0 = Mono sound -* 1 = Stereo sound -*/ -enum SrsCodecAudioSoundType -{ - // set to the max value to reserved, for array map. - SrsCodecAudioSoundTypeReserved = 2, - - SrsCodecAudioSoundTypeMono = 0, - SrsCodecAudioSoundTypeStereo = 1, -}; - -/** -* the codec sample unit. -* for h.264 video packet, a NALU is a sample unit. -* for aac raw audio packet, a NALU is the entire aac raw data. -* for sequence header, it's not a sample unit. -*/ -class SrsCodecSampleUnit -{ -public: - /** - * the sample bytes is directly ptr to packet bytes, - * user should never use it when packet destroyed. - */ - int size; - char* bytes; -public: - SrsCodecSampleUnit(); - virtual ~SrsCodecSampleUnit(); -}; - -/** -* the samples in the flv audio/video packet. -* the sample used to analysis a video/audio packet, -* split the h.264 NALUs to buffers, or aac raw data to a buffer, -* and decode the video/audio specified infos. -* -* the sample unit: -* a video packet codec in h.264 contains many NALUs, each is a sample unit. -* a audio packet codec in aac is a sample unit. -* @remark, the video/audio sequence header is not sample unit, -* all sequence header stores as extra data, -* @see SrsAvcAacCodec.avc_extra_data and SrsAvcAacCodec.aac_extra_data -* @remark, user must clear all samples before decode a new video/audio packet. -*/ -class SrsCodecSample -{ -public: - /** - * each audio/video raw data packet will dumps to one or multiple buffers, - * the buffers will write to hls and clear to reset. - * generally, aac audio packet corresponding to one buffer, - * where avc/h264 video packet may contains multiple buffer. - */ - int nb_sample_units; - SrsCodecSampleUnit sample_units[__SRS_SRS_MAX_CODEC_SAMPLE]; -public: - /** - * whether the sample is video sample which demux from video packet. - */ - bool is_video; - /** - * CompositionTime, video_file_format_spec_v10_1.pdf, page 78. - * cts = pts - dts, where dts = flvheader->timestamp. - */ - int32_t cts; -public: - // video specified - SrsCodecVideoAVCFrame frame_type; - SrsCodecVideoAVCType avc_packet_type; -public: - // audio specified - SrsCodecAudio acodec; - // audio aac specified. - SrsCodecAudioSampleRate sound_rate; - SrsCodecAudioSampleSize sound_size; - SrsCodecAudioSoundType sound_type; - SrsCodecAudioType aac_packet_type; -public: - SrsCodecSample(); - virtual ~SrsCodecSample(); -public: - /** - * clear all samples. - * the sample units never copy the bytes, it directly use the ptr, - * so when video/audio packet is destroyed, the sample must be clear. - * in a word, user must clear sample before demux it. - * @remark demux sample use SrsAvcAacCodec.audio_aac_demux or video_avc_demux. - */ - void clear(); - /** - * add the a sample unit, it's a h.264 NALU or aac raw data. - * the sample unit directly use the ptr of packet bytes, - * so user must never use sample unit when packet is destroyed. - * in a word, user must clear sample before demux it. - */ - int add_sample_unit(char* bytes, int size); -}; - -/** -* the h264/avc and aac codec, for media stream. -* -* to demux the FLV/RTMP video/audio packet to sample, -* add each NALUs of h.264 as a sample unit to sample, -* while the entire aac raw data as a sample unit. -* -* for sequence header, -* demux it and save it in the avc_extra_data and aac_extra_data, -* -* for the codec info, such as audio sample rate, -* decode from FLV/RTMP header, then use codec info in sequence -* header to override it. -*/ -class SrsAvcAacCodec -{ -private: - SrsStream* stream; -public: - /** - * metadata specified - */ - int duration; - int width; - int height; - int frame_rate; - // @see: SrsCodecVideo - int video_codec_id; - int video_data_rate; // in bps - // @see: SrsCod ecAudioType - int audio_codec_id; - int audio_data_rate; // in bps -public: - /** - * video specified - */ - // profile_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45. - u_int8_t avc_profile; - // level_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45. - u_int8_t avc_level; - // lengthSizeMinusOne, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 - int8_t NAL_unit_length; - u_int16_t sequenceParameterSetLength; - char* sequenceParameterSetNALUnit; - u_int16_t pictureParameterSetLength; - char* pictureParameterSetNALUnit; -public: - /** - * audio specified - * audioObjectType, in 1.6.2.1 AudioSpecificConfig, page 33, - * 1.5.1.1 Audio object type definition, page 23, - * in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf. - */ - u_int8_t aac_profile; - /** - * samplingFrequencyIndex - */ - u_int8_t aac_sample_rate; - /** - * channelConfiguration - */ - u_int8_t aac_channels; -public: - /** - * the avc extra data, the AVC sequence header, - * without the flv codec header, - * @see: ffmpeg, AVCodecContext::extradata - */ - int avc_extra_size; - char* avc_extra_data; - /** - * the aac extra data, the AAC sequence header, - * without the flv codec header, - * @see: ffmpeg, AVCodecContext::extradata - */ - int aac_extra_size; - char* aac_extra_data; -public: - SrsAvcAacCodec(); - virtual ~SrsAvcAacCodec(); -// the following function used for hls to build the sample and codec. -public: - /** - * demux the audio packet in aac codec. - * the packet mux in FLV/RTMP format defined in flv specification. - * demux the audio speicified data(sound_format, sound_size, ...) to sample. - * demux the aac specified data(aac_profile, ...) to codec from sequence header. - * demux the aac raw to sample units. - */ - virtual int audio_aac_demux(char* data, int size, SrsCodecSample* sample); - virtual int audio_mp3_demux(char* data, int size, SrsCodecSample* sample); - /** - * demux the video packet in h.264 codec. - * the packet mux in FLV/RTMP format defined in flv specification. - * demux the video specified data(frame_type, codec_id, ...) to sample. - * demux the h.264 sepcified data(avc_profile, ...) to codec from sequence header. - * demux the h.264 NALUs to sampe units. - */ - virtual int video_avc_demux(char* data, int size, SrsCodecSample* sample); -private: - /** - * when avc packet type is SrsCodecVideoAVCTypeSequenceHeader, - * decode the sps and pps. - */ - virtual int avc_demux_sps_pps(SrsStream* stream); - /** - * demux the avc NALU in "AnnexB" - * from H.264-AVC-ISO_IEC_14496-10.pdf, page 211. - */ - virtual int avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample); - /** - * demux the avc NALU in "ISO Base Media File Format" - * from H.264-AVC-ISO_IEC_14496-15.pdf, page 20 - */ - virtual int avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample); -}; - -#endif - diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index b9dbf6c72..4f4ada27b 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -25,6 +25,12 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include #include +using namespace std; + +#include +#include +#include +#include SrsFlvCodec::SrsFlvCodec() { @@ -111,3 +117,581 @@ bool SrsFlvCodec::audio_is_aac(char* data, int size) return sound_format == SrsCodecAudioAAC; } +SrsCodecSampleUnit::SrsCodecSampleUnit() +{ + size = 0; + bytes = NULL; +} + +SrsCodecSampleUnit::~SrsCodecSampleUnit() +{ +} + +SrsCodecSample::SrsCodecSample() +{ + clear(); +} + +SrsCodecSample::~SrsCodecSample() +{ +} + +void SrsCodecSample::clear() +{ + is_video = false; + nb_sample_units = 0; + + cts = 0; + frame_type = SrsCodecVideoAVCFrameReserved; + avc_packet_type = SrsCodecVideoAVCTypeReserved; + + acodec = SrsCodecAudioReserved1; + sound_rate = SrsCodecAudioSampleRateReserved; + sound_size = SrsCodecAudioSampleSizeReserved; + sound_type = SrsCodecAudioSoundTypeReserved; + aac_packet_type = SrsCodecAudioTypeReserved; +} + +int SrsCodecSample::add_sample_unit(char* bytes, int size) +{ + int ret = ERROR_SUCCESS; + + if (nb_sample_units >= __SRS_SRS_MAX_CODEC_SAMPLE) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("hls decode samples error, " + "exceed the max count: %d, ret=%d", __SRS_SRS_MAX_CODEC_SAMPLE, ret); + return ret; + } + + SrsCodecSampleUnit* sample_unit = &sample_units[nb_sample_units++]; + sample_unit->bytes = bytes; + sample_unit->size = size; + + return ret; +} + +SrsAvcAacCodec::SrsAvcAacCodec() +{ + width = 0; + height = 0; + duration = 0; + NAL_unit_length = 0; + frame_rate = 0; + + video_data_rate = 0; + video_codec_id = 0; + + audio_data_rate = 0; + audio_codec_id = 0; + + avc_profile = 0; + avc_level = 0; + aac_profile = 0; + aac_sample_rate = __SRS_AAC_SAMPLE_RATE_UNSET; // sample rate ignored + aac_channels = 0; + avc_extra_size = 0; + avc_extra_data = NULL; + aac_extra_size = 0; + aac_extra_data = NULL; + + sequenceParameterSetLength = 0; + sequenceParameterSetNALUnit = NULL; + pictureParameterSetLength = 0; + pictureParameterSetNALUnit = NULL; + + stream = new SrsStream(); +} + +SrsAvcAacCodec::~SrsAvcAacCodec() +{ + srs_freep(avc_extra_data); + srs_freep(aac_extra_data); + + srs_freep(stream); + srs_freep(sequenceParameterSetNALUnit); + srs_freep(pictureParameterSetNALUnit); +} + +int SrsAvcAacCodec::audio_aac_demux(char* data, int size, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + sample->is_video = false; + + if (!data || size <= 0) { + srs_trace("no audio present, ignore it."); + return ret; + } + + if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) { + return ret; + } + + // audio decode + if (!stream->require(1)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("audio codec decode sound_format failed. ret=%d", ret); + return ret; + } + + // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76 + int8_t sound_format = stream->read_1bytes(); + + int8_t sound_type = sound_format & 0x01; + int8_t sound_size = (sound_format >> 1) & 0x01; + int8_t sound_rate = (sound_format >> 2) & 0x03; + sound_format = (sound_format >> 4) & 0x0f; + + audio_codec_id = sound_format; + sample->acodec = (SrsCodecAudio)audio_codec_id; + + sample->sound_type = (SrsCodecAudioSoundType)sound_type; + sample->sound_rate = (SrsCodecAudioSampleRate)sound_rate; + sample->sound_size = (SrsCodecAudioSampleSize)sound_size; + + // we support h.264+mp3 for hls. + if (audio_codec_id == SrsCodecAudioMP3) { + return ERROR_HLS_TRY_MP3; + } + + // only support aac + if (audio_codec_id != SrsCodecAudioAAC) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("audio codec only support mp3/aac codec. actual=%d, ret=%d", audio_codec_id, ret); + return ret; + } + + if (!stream->require(1)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("audio codec decode aac_packet_type failed. ret=%d", ret); + return ret; + } + + int8_t aac_packet_type = stream->read_1bytes(); + sample->aac_packet_type = (SrsCodecAudioType)aac_packet_type; + + if (aac_packet_type == SrsCodecAudioTypeSequenceHeader) { + // AudioSpecificConfig + // 1.6.2.1 AudioSpecificConfig, in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf, page 33. + aac_extra_size = stream->size() - stream->pos(); + if (aac_extra_size > 0) { + srs_freep(aac_extra_data); + aac_extra_data = new char[aac_extra_size]; + memcpy(aac_extra_data, stream->data() + stream->pos(), aac_extra_size); + } + + // only need to decode the first 2bytes: + // audioObjectType, aac_profile, 5bits. + // samplingFrequencyIndex, aac_sample_rate, 4bits. + // channelConfiguration, aac_channels, 4bits + if (!stream->require(2)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("audio codec decode aac sequence header failed. ret=%d", ret); + return ret; + } + aac_profile = stream->read_1bytes(); + aac_sample_rate = stream->read_1bytes(); + + aac_channels = (aac_sample_rate >> 3) & 0x0f; + aac_sample_rate = ((aac_profile << 1) & 0x0e) | ((aac_sample_rate >> 7) & 0x01); + aac_profile = (aac_profile >> 3) & 0x1f; + + if (aac_profile == 0 || aac_profile == 0x1f) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("audio codec decode aac sequence header failed, " + "adts object=%d invalid. ret=%d", aac_profile, ret); + return ret; + } + + // TODO: FIXME: to support aac he/he-v2, see: ngx_rtmp_codec_parse_aac_header + // @see: https://github.com/winlinvip/nginx-rtmp-module/commit/3a5f9eea78fc8d11e8be922aea9ac349b9dcbfc2 + // + // donot force to LC, @see: https://github.com/winlinvip/simple-rtmp-server/issues/81 + // the source will print the sequence header info. + //if (aac_profile > 3) { + // Mark all extended profiles as LC + // to make Android as happy as possible. + // @see: ngx_rtmp_hls_parse_aac_header + //aac_profile = 1; + //} + } else if (aac_packet_type == SrsCodecAudioTypeRawData) { + // ensure the sequence header demuxed + if (aac_extra_size <= 0 || !aac_extra_data) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("audio codec decode aac failed, sequence header not found. ret=%d", ret); + return ret; + } + + // Raw AAC frame data in UI8 [] + // 6.3 Raw Data, aac-iso-13818-7.pdf, page 28 + if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), stream->size() - stream->pos())) != ERROR_SUCCESS) { + srs_error("audio codec add sample failed. ret=%d", ret); + return ret; + } + } else { + // ignored. + } + + // reset the sample rate by sequence header + if (aac_sample_rate != __SRS_AAC_SAMPLE_RATE_UNSET) { + static int aac_sample_rates[] = { + 96000, 88200, 64000, 48000, + 44100, 32000, 24000, 22050, + 16000, 12000, 11025, 8000, + 7350, 0, 0, 0 + }; + switch (aac_sample_rates[aac_sample_rate]) { + case 11025: + sample->sound_rate = SrsCodecAudioSampleRate11025; + break; + case 22050: + sample->sound_rate = SrsCodecAudioSampleRate22050; + break; + case 44100: + sample->sound_rate = SrsCodecAudioSampleRate44100; + break; + default: + break; + }; + } + + srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", + sound_type, audio_codec_id, sound_size, sound_rate, sound_format, size); + + return ret; +} + +int SrsAvcAacCodec::audio_mp3_demux(char* data, int size, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + // we always decode aac then mp3. + srs_assert(sample->acodec == SrsCodecAudioMP3); + + // @see: E.4.2 Audio Tags, video_file_format_spec_v10_1.pdf, page 76 + if (!data || size <= 1) { + srs_trace("no mp3 audio present, ignore it."); + return ret; + } + + // mp3 payload. + if ((ret = sample->add_sample_unit(data + 1, size - 1)) != ERROR_SUCCESS) { + srs_error("audio codec add mp3 sample failed. ret=%d", ret); + return ret; + } + + srs_info("audio decoded, type=%d, codec=%d, asize=%d, rate=%d, format=%d, size=%d", + sample->sound_type, audio_codec_id, sample->sound_size, sample->sound_rate, sample->acodec, size); + + return ret; +} + +int SrsAvcAacCodec::video_avc_demux(char* data, int size, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + sample->is_video = true; + + if (!data || size <= 0) { + srs_trace("no video present, ignore it."); + return ret; + } + + if ((ret = stream->initialize(data, size)) != ERROR_SUCCESS) { + return ret; + } + + // video decode + if (!stream->require(1)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("video codec decode frame_type failed. ret=%d", ret); + return ret; + } + + // @see: E.4.3 Video Tags, video_file_format_spec_v10_1.pdf, page 78 + int8_t frame_type = stream->read_1bytes(); + int8_t codec_id = frame_type & 0x0f; + frame_type = (frame_type >> 4) & 0x0f; + + sample->frame_type = (SrsCodecVideoAVCFrame)frame_type; + + // ignore info frame without error, + // @see https://github.com/winlinvip/simple-rtmp-server/issues/288#issuecomment-69863909 + if (sample->frame_type == SrsCodecVideoAVCFrameVideoInfoFrame) { + srs_warn("video codec igone the info frame, ret=%d", ret); + return ret; + } + + // only support h.264/avc + if (codec_id != SrsCodecVideoAVC) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("video codec only support video h.264/avc codec. actual=%d, ret=%d", codec_id, ret); + return ret; + } + video_codec_id = codec_id; + + if (!stream->require(4)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("video codec decode avc_packet_type failed. ret=%d", ret); + return ret; + } + int8_t avc_packet_type = stream->read_1bytes(); + int32_t composition_time = stream->read_3bytes(); + + // pts = dts + cts. + sample->cts = composition_time; + sample->avc_packet_type = (SrsCodecVideoAVCType)avc_packet_type; + + if (avc_packet_type == SrsCodecVideoAVCTypeSequenceHeader) { + if ((ret = avc_demux_sps_pps(stream)) != ERROR_SUCCESS) { + return ret; + } + } else if (avc_packet_type == SrsCodecVideoAVCTypeNALU){ + // ensure the sequence header demuxed + if (avc_extra_size <= 0 || !avc_extra_data) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode failed, sequence header not found. ret=%d", ret); + return ret; + } + + // One or more NALUs (Full frames are required) + // try "AnnexB" from H.264-AVC-ISO_IEC_14496-10.pdf, page 211. + if ((ret = avc_demux_annexb_format(stream, sample)) != ERROR_SUCCESS) { + // stop try when system error. + if (ret != ERROR_HLS_AVC_TRY_OTHERS) { + srs_error("avc demux for annexb failed. ret=%d", ret); + return ret; + } + + // try "ISO Base Media File Format" from H.264-AVC-ISO_IEC_14496-15.pdf, page 20 + if ((ret = avc_demux_ibmf_format(stream, sample)) != ERROR_SUCCESS) { + return ret; + } + } + } else { + // ignored. + } + + srs_info("video decoded, type=%d, codec=%d, avc=%d, time=%d, size=%d", + frame_type, video_codec_id, avc_packet_type, composition_time, size); + + return ret; +} + +int SrsAvcAacCodec::avc_demux_sps_pps(SrsStream* stream) +{ + int ret = ERROR_SUCCESS; + + // AVCDecoderConfigurationRecord + // 5.2.4.1.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 + avc_extra_size = stream->size() - stream->pos(); + if (avc_extra_size > 0) { + srs_freep(avc_extra_data); + avc_extra_data = new char[avc_extra_size]; + memcpy(avc_extra_data, stream->data() + stream->pos(), avc_extra_size); + } + + if (!stream->require(6)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header failed. ret=%d", ret); + return ret; + } + //int8_t configurationVersion = stream->read_1bytes(); + stream->read_1bytes(); + //int8_t AVCProfileIndication = stream->read_1bytes(); + avc_profile = stream->read_1bytes(); + //int8_t profile_compatibility = stream->read_1bytes(); + stream->read_1bytes(); + //int8_t AVCLevelIndication = stream->read_1bytes(); + avc_level = stream->read_1bytes(); + + // parse the NALU size. + int8_t lengthSizeMinusOne = stream->read_1bytes(); + lengthSizeMinusOne &= 0x03; + NAL_unit_length = lengthSizeMinusOne; + + // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 + // 5.2.4.1 AVC decoder configuration record + // 5.2.4.1.2 Semantics + // The value of this field shall be one of 0, 1, or 3 corresponding to a + // length encoded with 1, 2, or 4 bytes, respectively. + if (NAL_unit_length == 2) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("sps lengthSizeMinusOne should never be 2. ret=%d", ret); + return ret; + } + + // 1 sps + if (!stream->require(1)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header sps failed. ret=%d", ret); + return ret; + } + int8_t numOfSequenceParameterSets = stream->read_1bytes(); + numOfSequenceParameterSets &= 0x1f; + if (numOfSequenceParameterSets != 1) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header sps failed. ret=%d", ret); + return ret; + } + if (!stream->require(2)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header sps size failed. ret=%d", ret); + return ret; + } + sequenceParameterSetLength = stream->read_2bytes(); + if (!stream->require(sequenceParameterSetLength)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header sps data failed. ret=%d", ret); + return ret; + } + if (sequenceParameterSetLength > 0) { + srs_freep(sequenceParameterSetNALUnit); + sequenceParameterSetNALUnit = new char[sequenceParameterSetLength]; + memcpy(sequenceParameterSetNALUnit, stream->data() + stream->pos(), sequenceParameterSetLength); + stream->skip(sequenceParameterSetLength); + } + // 1 pps + if (!stream->require(1)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header pps failed. ret=%d", ret); + return ret; + } + int8_t numOfPictureParameterSets = stream->read_1bytes(); + numOfPictureParameterSets &= 0x1f; + if (numOfPictureParameterSets != 1) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header pps failed. ret=%d", ret); + return ret; + } + if (!stream->require(2)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header pps size failed. ret=%d", ret); + return ret; + } + pictureParameterSetLength = stream->read_2bytes(); + if (!stream->require(pictureParameterSetLength)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode sequenc header pps data failed. ret=%d", ret); + return ret; + } + if (pictureParameterSetLength > 0) { + srs_freep(pictureParameterSetNALUnit); + pictureParameterSetNALUnit = new char[pictureParameterSetLength]; + memcpy(pictureParameterSetNALUnit, stream->data() + stream->pos(), pictureParameterSetLength); + stream->skip(pictureParameterSetLength); + } + + return ret; +} + +int SrsAvcAacCodec::avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + // not annexb, try others + if (!srs_avc_startswith_annexb(stream, NULL)) { + return ERROR_HLS_AVC_TRY_OTHERS; + } + + // AnnexB + // B.1.1 Byte stream NAL unit syntax, + // H.264-AVC-ISO_IEC_14496-10.pdf, page 211. + while (!stream->empty()) { + // find start code + int nb_start_code = 0; + if (!srs_avc_startswith_annexb(stream, &nb_start_code)) { + return ret; + } + + // skip the start code. + if (nb_start_code > 0) { + stream->skip(nb_start_code); + } + + // the NALU start bytes. + char* p = stream->data() + stream->pos(); + + // get the last matched NALU + while (!stream->empty()) { + if (srs_avc_startswith_annexb(stream, NULL)) { + break; + } + + stream->skip(1); + } + + char* pp = stream->data() + stream->pos(); + + // skip the empty. + if (pp - p <= 0) { + continue; + } + + // got the NALU. + if ((ret = sample->add_sample_unit(p, pp - p)) != ERROR_SUCCESS) { + srs_error("annexb add video sample failed. ret=%d", ret); + return ret; + } + } + + return ret; +} + +int SrsAvcAacCodec::avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample) +{ + int ret = ERROR_SUCCESS; + + int PictureLength = stream->size() - stream->pos(); + + // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 + // 5.2.4.1 AVC decoder configuration record + // 5.2.4.1.2 Semantics + // The value of this field shall be one of 0, 1, or 3 corresponding to a + // length encoded with 1, 2, or 4 bytes, respectively. + srs_assert(NAL_unit_length != 2); + + // 5.3.4.2.1 Syntax, H.264-AVC-ISO_IEC_14496-15.pdf, page 20 + for (int i = 0; i < PictureLength;) { + // unsigned int((NAL_unit_length+1)*8) NALUnitLength; + if (!stream->require(NAL_unit_length + 1)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode NALU size failed. ret=%d", ret); + return ret; + } + int32_t NALUnitLength = 0; + if (NAL_unit_length == 3) { + NALUnitLength = stream->read_4bytes(); + } else if (NAL_unit_length == 1) { + NALUnitLength = stream->read_2bytes(); + } else { + NALUnitLength = stream->read_1bytes(); + } + + // maybe stream is invalid format. + // see: https://github.com/winlinvip/simple-rtmp-server/issues/183 + if (NALUnitLength < 0) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("maybe stream is AnnexB format. ret=%d", ret); + return ret; + } + + // NALUnit + if (!stream->require(NALUnitLength)) { + ret = ERROR_HLS_DECODE_ERROR; + srs_error("avc decode NALU data failed. ret=%d", ret); + return ret; + } + // 7.3.1 NAL unit syntax, H.264-AVC-ISO_IEC_14496-10.pdf, page 44. + if ((ret = sample->add_sample_unit(stream->data() + stream->pos(), NALUnitLength)) != ERROR_SUCCESS) { + srs_error("avc add video sample failed. ret=%d", ret); + return ret; + } + stream->skip(NALUnitLength); + + i += NAL_unit_length + 1 + NALUnitLength; + } + + return ret; +} + diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 8ee98f3e5..e41793dcc 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -30,6 +30,10 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. #include +#include + +class SrsStream; + // AACPacketType IF SoundFormat == 10 UI8 // The following values are defined: // 0 = AAC sequence header @@ -212,4 +216,263 @@ public: static bool audio_is_aac(char* data, int size); }; +/** +* the public data, event HLS disable, others can use it. +*/ +/** +* the flv sample rate map +*/ +extern int flv_sample_rates[]; + +/** +* the aac sample rate map +*/ +extern int aac_sample_rates[]; + +#define __SRS_SRS_MAX_CODEC_SAMPLE 128 +#define __SRS_AAC_SAMPLE_RATE_UNSET 15 + +// in ms, for HLS aac flush the audio +#define SRS_CONF_DEFAULT_AAC_DELAY 100 + +// max PES packets size to flush the video. +#define SRS_AUTO_HLS_AUDIO_CACHE_SIZE 1024 * 1024 + +/** +* the FLV/RTMP supported audio sample size. +* Size of each audio sample. This parameter only pertains to +* uncompressed formats. Compressed formats always decode +* to 16 bits internally. +* 0 = 8-bit samples +* 1 = 16-bit samples +*/ +enum SrsCodecAudioSampleSize +{ + // set to the max value to reserved, for array map. + SrsCodecAudioSampleSizeReserved = 2, + + SrsCodecAudioSampleSize8bit = 0, + SrsCodecAudioSampleSize16bit = 1, +}; + +/** +* the FLV/RTMP supported audio sound type/channel. +* Mono or stereo sound +* 0 = Mono sound +* 1 = Stereo sound +*/ +enum SrsCodecAudioSoundType +{ + // set to the max value to reserved, for array map. + SrsCodecAudioSoundTypeReserved = 2, + + SrsCodecAudioSoundTypeMono = 0, + SrsCodecAudioSoundTypeStereo = 1, +}; + +/** +* the codec sample unit. +* for h.264 video packet, a NALU is a sample unit. +* for aac raw audio packet, a NALU is the entire aac raw data. +* for sequence header, it's not a sample unit. +*/ +class SrsCodecSampleUnit +{ +public: + /** + * the sample bytes is directly ptr to packet bytes, + * user should never use it when packet destroyed. + */ + int size; + char* bytes; +public: + SrsCodecSampleUnit(); + virtual ~SrsCodecSampleUnit(); +}; + +/** +* the samples in the flv audio/video packet. +* the sample used to analysis a video/audio packet, +* split the h.264 NALUs to buffers, or aac raw data to a buffer, +* and decode the video/audio specified infos. +* +* the sample unit: +* a video packet codec in h.264 contains many NALUs, each is a sample unit. +* a audio packet codec in aac is a sample unit. +* @remark, the video/audio sequence header is not sample unit, +* all sequence header stores as extra data, +* @see SrsAvcAacCodec.avc_extra_data and SrsAvcAacCodec.aac_extra_data +* @remark, user must clear all samples before decode a new video/audio packet. +*/ +class SrsCodecSample +{ +public: + /** + * each audio/video raw data packet will dumps to one or multiple buffers, + * the buffers will write to hls and clear to reset. + * generally, aac audio packet corresponding to one buffer, + * where avc/h264 video packet may contains multiple buffer. + */ + int nb_sample_units; + SrsCodecSampleUnit sample_units[__SRS_SRS_MAX_CODEC_SAMPLE]; +public: + /** + * whether the sample is video sample which demux from video packet. + */ + bool is_video; + /** + * CompositionTime, video_file_format_spec_v10_1.pdf, page 78. + * cts = pts - dts, where dts = flvheader->timestamp. + */ + int32_t cts; +public: + // video specified + SrsCodecVideoAVCFrame frame_type; + SrsCodecVideoAVCType avc_packet_type; +public: + // audio specified + SrsCodecAudio acodec; + // audio aac specified. + SrsCodecAudioSampleRate sound_rate; + SrsCodecAudioSampleSize sound_size; + SrsCodecAudioSoundType sound_type; + SrsCodecAudioType aac_packet_type; +public: + SrsCodecSample(); + virtual ~SrsCodecSample(); +public: + /** + * clear all samples. + * the sample units never copy the bytes, it directly use the ptr, + * so when video/audio packet is destroyed, the sample must be clear. + * in a word, user must clear sample before demux it. + * @remark demux sample use SrsAvcAacCodec.audio_aac_demux or video_avc_demux. + */ + void clear(); + /** + * add the a sample unit, it's a h.264 NALU or aac raw data. + * the sample unit directly use the ptr of packet bytes, + * so user must never use sample unit when packet is destroyed. + * in a word, user must clear sample before demux it. + */ + int add_sample_unit(char* bytes, int size); +}; + +/** +* the h264/avc and aac codec, for media stream. +* +* to demux the FLV/RTMP video/audio packet to sample, +* add each NALUs of h.264 as a sample unit to sample, +* while the entire aac raw data as a sample unit. +* +* for sequence header, +* demux it and save it in the avc_extra_data and aac_extra_data, +* +* for the codec info, such as audio sample rate, +* decode from FLV/RTMP header, then use codec info in sequence +* header to override it. +*/ +class SrsAvcAacCodec +{ +private: + SrsStream* stream; +public: + /** + * metadata specified + */ + int duration; + int width; + int height; + int frame_rate; + // @see: SrsCodecVideo + int video_codec_id; + int video_data_rate; // in bps + // @see: SrsCod ecAudioType + int audio_codec_id; + int audio_data_rate; // in bps +public: + /** + * video specified + */ + // profile_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45. + u_int8_t avc_profile; + // level_idc, H.264-AVC-ISO_IEC_14496-10.pdf, page 45. + u_int8_t avc_level; + // lengthSizeMinusOne, H.264-AVC-ISO_IEC_14496-15.pdf, page 16 + int8_t NAL_unit_length; + u_int16_t sequenceParameterSetLength; + char* sequenceParameterSetNALUnit; + u_int16_t pictureParameterSetLength; + char* pictureParameterSetNALUnit; +public: + /** + * audio specified + * audioObjectType, in 1.6.2.1 AudioSpecificConfig, page 33, + * 1.5.1.1 Audio object type definition, page 23, + * in aac-mp4a-format-ISO_IEC_14496-3+2001.pdf. + */ + u_int8_t aac_profile; + /** + * samplingFrequencyIndex + */ + u_int8_t aac_sample_rate; + /** + * channelConfiguration + */ + u_int8_t aac_channels; +public: + /** + * the avc extra data, the AVC sequence header, + * without the flv codec header, + * @see: ffmpeg, AVCodecContext::extradata + */ + int avc_extra_size; + char* avc_extra_data; + /** + * the aac extra data, the AAC sequence header, + * without the flv codec header, + * @see: ffmpeg, AVCodecContext::extradata + */ + int aac_extra_size; + char* aac_extra_data; +public: + SrsAvcAacCodec(); + virtual ~SrsAvcAacCodec(); +// the following function used for hls to build the sample and codec. +public: + /** + * demux the audio packet in aac codec. + * the packet mux in FLV/RTMP format defined in flv specification. + * demux the audio speicified data(sound_format, sound_size, ...) to sample. + * demux the aac specified data(aac_profile, ...) to codec from sequence header. + * demux the aac raw to sample units. + */ + virtual int audio_aac_demux(char* data, int size, SrsCodecSample* sample); + virtual int audio_mp3_demux(char* data, int size, SrsCodecSample* sample); + /** + * demux the video packet in h.264 codec. + * the packet mux in FLV/RTMP format defined in flv specification. + * demux the video specified data(frame_type, codec_id, ...) to sample. + * demux the h.264 sepcified data(avc_profile, ...) to codec from sequence header. + * demux the h.264 NALUs to sampe units. + */ + virtual int video_avc_demux(char* data, int size, SrsCodecSample* sample); +private: + /** + * when avc packet type is SrsCodecVideoAVCTypeSequenceHeader, + * decode the sps and pps. + */ + virtual int avc_demux_sps_pps(SrsStream* stream); + /** + * demux the avc NALU in "AnnexB" + * from H.264-AVC-ISO_IEC_14496-10.pdf, page 211. + */ + virtual int avc_demux_annexb_format(SrsStream* stream, SrsCodecSample* sample); + /** + * demux the avc NALU in "ISO Base Media File Format" + * from H.264-AVC-ISO_IEC_14496-15.pdf, page 20 + */ + virtual int avc_demux_ibmf_format(SrsStream* stream, SrsCodecSample* sample); +}; + #endif diff --git a/trunk/src/kernel/srs_kernel_ts.cpp b/trunk/src/kernel/srs_kernel_ts.cpp index 26766d01b..439261172 100644 --- a/trunk/src/kernel/srs_kernel_ts.cpp +++ b/trunk/src/kernel/srs_kernel_ts.cpp @@ -35,7 +35,7 @@ using namespace std; #include #include #include -#include +#include #include #include #include diff --git a/trunk/src/libs/srs_librtmp.cpp b/trunk/src/libs/srs_librtmp.cpp index 90523e7dc..3a8bb5d23 100644 --- a/trunk/src/libs/srs_librtmp.cpp +++ b/trunk/src/libs/srs_librtmp.cpp @@ -1520,7 +1520,6 @@ int srs_h264_write_raw_frames(srs_rtmp_t rtmp, while (!context->h264_raw_stream.empty()) { char* frame = NULL; int frame_size = 0; - bool got_sps_pps = false; if ((ret = context->avc_raw.annexb_demux(&context->h264_raw_stream, &frame, &frame_size)) != ERROR_SUCCESS) { return ret; }