一：整体代码函数预览
static ngx_int_t
ngx_rtmp_mp4_postconfiguration(ngx_conf_t *cf)
{
ngx_rtmp_play_main_conf_t *pmcf;
ngx_rtmp_play_fmt_t **pfmt, *fmt;
pmcf = ngx_rtmp_conf_get_module_main_conf(cf, ngx_rtmp_play_module);
pfmt = ngx_array_push(&pmcf->fmts);
if (pfmt == NULL) {
return NGX_ERROR;
}
fmt = ngx_pcalloc(cf->pool, sizeof(ngx_rtmp_play_fmt_t));
if (fmt == NULL) {
return NGX_ERROR;
}
*pfmt = fmt;
ngx_str_set(&fmt->name, "mp4-format");
ngx_str_set(&fmt->pfx, "mp4:");
ngx_str_set(&fmt->sfx, ".mp4");
fmt->init = ngx_rtmp_mp4_init; // 初始化
fmt->done = ngx_rtmp_mp4_done; // 完成
fmt->seek = ngx_rtmp_mp4_seek; // seek
fmt->start = ngx_rtmp_mp4_start; // 开始
fmt->stop = ngx_rtmp_mp4_stop; // 结束
fmt->send = ngx_rtmp_mp4_send; // 发送数据
return NGX_OK;
}二：数据结构定义
|---------------------------------------|---------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| 定义 | 标准定义 | nginx 数据结构定义 |
| stsc 【Sample To Chunk Box】 | 
| type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t first_chunk; uint32_t samples_per_chunk; uint32_t sample_descrption_index; } ngx_rtmp_mp4_chunk_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_chunk_entry_t entries[0]; } ngx_rtmp_mp4_chunks_t; |
| stts 【Decoding Time to Sample Box】 | 
| type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t sample_count; uint32_t sample_delta; } ngx_rtmp_mp4_time_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_time_entry_t entries[0]; } ngx_rtmp_mp4_times_t; |
| ctts 【Composition Time to Sample Box】 | 
| type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t sample_count; uint32_t sample_offset; } ngx_rtmp_mp4_delay_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_delay_entry_t entries[0]; } ngx_rtmp_mp4_delays_t; |
| stss 【Sync Sample Box】 | 
| type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t version_flags; uint32_t entry_count; uint32_t entries[0]; } ngx_rtmp_mp4_keys_t; |
| stsz 【Sample Size Boxes】 | 
| type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t version_flags; 4 字节 uint32_t sample_size; uint32_t sample_count; uint32_t entries[0]; } ngx_rtmp_mp4_sizes_t; |
| stco 【Chunk Offset Box】 co64 | 
| type 4 字节已经偏移过， box_size 4 字节已经偏移过 typedef struct { uint32_t version_flags; 4 字节固定 uint32_t entry_count; uint32_t entries[0]; } ngx_rtmp_mp4_offsets_t; typedef struct { uint32_t version_flags; uint32_t entry_count; uint64_t entries[0]; } ngx_rtmp_mp4_offsets64_t; |
三：数据结构解析
static ngx_rtmp_mp4_box_t ngx_rtmp_mp4_boxes[] = {
{ ngx_rtmp_mp4_make_tag('t','r','a','k'), ngx_rtmp_mp4_parse_trak },
{ ngx_rtmp_mp4_make_tag('m','d','i','a'), ngx_rtmp_mp4_parse },
{ ngx_rtmp_mp4_make_tag('m','d','h','d'), ngx_rtmp_mp4_parse_mdhd },
{ ngx_rtmp_mp4_make_tag('h','d','l','r'), ngx_rtmp_mp4_parse_hdlr },
{ ngx_rtmp_mp4_make_tag('m','i','n','f'), ngx_rtmp_mp4_parse },
{ ngx_rtmp_mp4_make_tag('s','t','b','l'), ngx_rtmp_mp4_parse },
{ ngx_rtmp_mp4_make_tag('s','t','s','d'), ngx_rtmp_mp4_parse_stsd },
{ ngx_rtmp_mp4_make_tag('s','t','s','c'), ngx_rtmp_mp4_parse_stsc }, // 记录了每个chunk中包含多少sample
{ ngx_rtmp_mp4_make_tag('s','t','t','s'), ngx_rtmp_mp4_parse_stts }, // sample 解码时间的压缩表
{ ngx_rtmp_mp4_make_tag('c','t','t','s'), ngx_rtmp_mp4_parse_ctts }, // 帧解码到渲染的时间差值，通常用在B帧的场景，sample的CTS与DTS的时间差的压缩表
{ ngx_rtmp_mp4_make_tag('s','t','s','s'), ngx_rtmp_mp4_parse_stss }, // 关键帧映射表
{ ngx_rtmp_mp4_make_tag('s','t','s','z'), ngx_rtmp_mp4_parse_stsz }, // 每帧数据的大小
{ ngx_rtmp_mp4_make_tag('s','t','z','2'), ngx_rtmp_mp4_parse_stz2 },
{ ngx_rtmp_mp4_make_tag('s','t','c','o'), ngx_rtmp_mp4_parse_stco }, // 记录了chunk对应的offset
{ ngx_rtmp_mp4_make_tag('c','o','6','4'), ngx_rtmp_mp4_parse_co64 },
{ ngx_rtmp_mp4_make_tag('a','v','c','1'), ngx_rtmp_mp4_parse_avc1 },
{ ngx_rtmp_mp4_make_tag('a','v','c','C'), ngx_rtmp_mp4_parse_avcC }, // sps pps 解析
{ ngx_rtmp_mp4_make_tag('m','p','4','a'), ngx_rtmp_mp4_parse_mp4a },
{ ngx_rtmp_mp4_make_tag('m','p','4','v'), ngx_rtmp_mp4_parse_mp4v },
{ ngx_rtmp_mp4_make_tag('e','s','d','s'), ngx_rtmp_mp4_parse_esds },
{ ngx_rtmp_mp4_make_tag('.','m','p','3'), ngx_rtmp_mp4_parse_mp3 },
{ ngx_rtmp_mp4_make_tag('n','m','o','s'), ngx_rtmp_mp4_parse_nmos },
{ ngx_rtmp_mp4_make_tag('s','p','e','x'), ngx_rtmp_mp4_parse_spex },
{ ngx_rtmp_mp4_make_tag('w','a','v','e'), ngx_rtmp_mp4_parse }
};ngx_rtmp_mp4_parse_stsc 记录了每个chunk中包含多少sample
上图数据解析
00 00 00 28(size) --> 40 字节总大小包含的自己在内的所有数据：
00 00 00 28(size) 73 74 73 63（stsc） 00 00 00 00(version_flags) 00 00 00 02(entry_count) 00 00 00 01(first_chunk) 00 00 00 02(samples_per_chunk) 00 00 00 01(sample_descrption_index) 00 00 00 02(first_chunk) 00 00 00 01(samples_per_chunk) 00 00 00 01(sample_descrption_index)
static ngx_int_t
ngx_rtmp_mp4_parse_stsc(ngx_rtmp_session_t *s, u_char *pos, u_char *last)
{
ngx_rtmp_mp4_ctx_t *ctx;
ngx_rtmp_mp4_track_t *t;
ctx = ngx_rtmp_get_module_ctx(s, ngx_rtmp_mp4_module);
t = ctx->track;
if (t == NULL) {
return NGX_OK;
}
t->chunks = (ngx_rtmp_mp4_chunks_t *) pos; // 内存数据上面的解析
if (pos + sizeof(*t->chunks) + ngx_rtmp_r32(t->chunks->entry_count) *
sizeof(t->chunks->entries[0])
<= last)// 内存越界判断
{
return NGX_OK;
}
t->chunks = NULL;
return NGX_ERROR;
}ngx_rtmp_mp4_parse_stco 记录了chunk对应的offset
00 00 03 D4（980 长度） 73 74 63 6F （stco）00 00 00 00(version_flags) 00 00 00 F1( entry_count 241 ) 00 00 00 30( entrie) 00 00 10 4D ( entrie)00 00 20 1C( entrie) 00 00 29 38( entrie) 00 00 49 2D( entrie) 00 00 70 C9( entrie) 00 00 B1 C1 ( entrie) ...............
static ngx_int_t
ngx_rtmp_mp4_parse_stco(ngx_rtmp_session_t *s, u_char *pos, u_char *last)
{
ngx_rtmp_mp4_ctx_t *ctx;
ngx_rtmp_mp4_track_t *t;
ctx = ngx_rtmp_get_module_ctx(s, ngx_rtmp_mp4_module);
t = ctx->track;
if (t == NULL) {
return NGX_OK;
}
t->offsets = (ngx_rtmp_mp4_offsets_t *) pos; // 数据结构的赋值
if (pos + sizeof(*t->offsets) + ngx_rtmp_r32(t->offsets->entry_count) *
sizeof(t->offsets->entries[0])
<= last) // 内存越界判断
{
return NGX_OK;
}
t->offsets = NULL;
return NGX_ERROR;
}其他数据结构的解析与上面的流程基本一样
四：数据处理流程
五：play 流程代码分析
ffplay -analyzeduration 1000000 "rtmp://10.90.103.5/vod/0001.mp4
1> ngx_rtmp_mp4_init 初始化变量解析 mp4 moov box 的数据结构
2> ngx_rtmp_mp4_seek (ngx_rtmp_session_t *s, ngx_file_t *f, ngx_uint_t timestamp)， timestamp = 0 ，seek 时间戳为零标识从头开始播放。 ngx_rtmp_mp4_seek_track 有两个轨道，音频轨和视频轨，分别进行seek
static ngx_int_t
ngx_rtmp_mp4_seek_track(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t,
ngx_int_t timestamp)
{
ngx_rtmp_mp4_cursor_t *cr;
cr = &t->cursor;
ngx_memzero(cr, sizeof(*cr));
if (ngx_rtmp_mp4_seek_time(s, t, ngx_rtmp_mp4_from_rtmp_timestamp( // ngx_rtmp_mp4_from_rtmp_timestamp 时间戳的
t, timestamp)) != NGX_OK ||
ngx_rtmp_mp4_seek_key(s, t) != NGX_OK ||
ngx_rtmp_mp4_seek_chunk(s, t) != NGX_OK ||
ngx_rtmp_mp4_seek_size(s, t) != NGX_OK ||
ngx_rtmp_mp4_seek_delay(s, t) != NGX_OK)
{
return NGX_ERROR;
}
cr->valid = 1;
return NGX_OK;
}
// 时间戳的转换函数，mp4 文件到rtmp
// rtmp_mp4 存放的是timescale 时间戳 通常是 12288
// rtmp 时间刻度通常是 1000
static ngx_inline uint32_t
ngx_rtmp_mp4_to_rtmp_timestamp(ngx_rtmp_mp4_track_t *t, uint64_t ts) // 给定一个时间值 从rtmp_mp4 12288 刻度转换到 1000
{
return (uint32_t) (ts * 1000 / t->time_scale);
}
static ngx_inline uint32_t
ngx_rtmp_mp4_from_rtmp_timestamp(ngx_rtmp_mp4_track_t *t, uint32_t ts) // 给定一个时间值 从rtmp 1000 刻度转换到12288
{
return (uint64_t) ts * t->time_scale / 1000;
}ngx_rtmp_mp4_seek_time :
|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|
| stts： typedef struct { uint32_t sample_count; uint32_t sample_delta; } ngx_rtmp_mp4_time_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_time_entry_t entries[0]; } ngx_rtmp_mp4_times_t; | 
| 
|
static ngx_int_t
ngx_rtmp_mp4_seek_time(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t,
uint32_t timestamp)
{
ngx_rtmp_mp4_cursor_t *cr;
ngx_rtmp_mp4_time_entry_t *te;
uint32_t dt;
if (t->times == NULL) {
return NGX_ERROR;
}
cr = &t->cursor; // 对应音频或者视频轨道的游标器
te = t->times->entries; // times --> ngx_rtmp_mp4_times_t
// sample_counts 6,1,5,1,5,1,1,1,1,1
// sample_deltas 1024,3040,1024,3088,1024,1008,1024,1032,1024,1048
// 整个循环就是处理，遍历sample_counts个sample_deltas值与 参数 timestamp比大小. 1. 小于记录 cr->timestamp += dt,
while (cr->time_pos < ngx_rtmp_r32(t->times->entry_count)) {
dt = ngx_rtmp_r32(te->sample_delta) * ngx_rtmp_r32(te->sample_count); // 242 * 1024 = 22757376
// cr->timestamp + dt >= timestamp ，在这个范围说明找到
if (cr->timestamp + dt >= timestamp) {
if (te->sample_delta == 0) {
return NGX_ERROR;
}
cr->time_count = (timestamp - cr->timestamp) /
ngx_rtmp_r32(te->sample_delta); // sample_delta 相同，cr->time_count 等于当前时间戳距离这个entry开始时间错的插值，的第几个delta
cr->timestamp += ngx_rtmp_r32(te->sample_delta) * cr->time_count; // seek 的时间戳，就是把 cr->timestamp 修改当前的时间戳
cr->pos += cr->time_count; // 记录当前seek 对应时间戳具体的位置
break;
}
// 没找到，
cr->timestamp += dt; // 记录值进行累加，访问下一个
cr->pos += ngx_rtmp_r32(te->sample_count); // 记录pos 的位置
cr->time_pos++; // 记录已经访问第几个entry
te++; // te++ 下一个entry
}
if (cr->time_pos >= ngx_rtmp_r32(t->times->entry_count)) {
return NGX_ERROR;
}
return NGX_OK;
}ngx_rtmp_mp4_seek_key
|--------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|---|
| stss typedef struct { uint32_t version_flags; uint32_t entry_count; // 个数 uint32_t entries[0]; // pos } ngx_rtmp_mp4_keys_t; | 
| |
static ngx_int_t
ngx_rtmp_mp4_seek_key(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{
ngx_rtmp_mp4_cursor_t *cr;
uint32_t *ke;
ngx_int_t dpos;
cr = &t->cursor; // 对应音频或者视频轨道的游标器
if (t->keys == NULL) {
return NGX_OK;
}
// cr->key_pos 初始值为0 ，t->keys->entry_count 关键帧的个数
while (cr->key_pos < ngx_rtmp_r32(t->keys->entry_count)) {
// 遍历每一个entry得到帧号，与当前pos位置进行对比，找到当前位置的下一个关键帧
if (ngx_rtmp_r32(t->keys->entries[cr->key_pos]) > cr->pos) { // cr->pos 上面seek 后具体位置
break;
}
cr->key_pos++;
}
if (cr->key_pos >= ngx_rtmp_r32(t->keys->entry_count)) {
return NGX_OK;
}
ke = &t->keys->entries[cr->key_pos]; //
dpos = ngx_rtmp_r32(*ke) - cr->pos - 1; // 计算距离下一个关键帧的差值
cr->key = 1;
/* TODO: range version needed */
for (; dpos > 0; --dpos) {
ngx_rtmp_mp4_next_time(s, t);
}
return NGX_OK;
}ngx_rtmp_mp4_seek_chunk
|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|
| stsc typedef struct { uint32_t first_chunk; uint32_t samples_per_chunk; uint32_t sample_descrption_index; } ngx_rtmp_mp4_chunk_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_chunk_entry_t entries[0]; } ngx_rtmp_mp4_chunks_t; | 
| 
|
static ngx_int_t
ngx_rtmp_mp4_seek_chunk(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{
ngx_rtmp_mp4_cursor_t *cr;
ngx_rtmp_mp4_chunk_entry_t *ce, *nce;
ngx_uint_t pos, dpos, dchunk;
cr = &t->cursor; // 对应音频或者视频轨道的游标器
if (t->chunks == NULL || t->chunks->entry_count == 0) {
cr->chunk = 1;
return NGX_OK;
}
ce = t->chunks->entries; // entry 实体结构
pos = 0;
// cr->chunk_pos 初始值是0
while (cr->chunk_pos + 1 < ngx_rtmp_r32(t->chunks->entry_count)) {
nce = ce + 1; // 下一个entry
dpos = (ngx_rtmp_r32(nce->first_chunk) -
ngx_rtmp_r32(ce->first_chunk)) *
ngx_rtmp_r32(ce->samples_per_chunk); // 距离下一个chunk总共有多少点，（2-1） * 2 = 2 帧
if (pos + dpos > cr->pos) { // 判断当前位置是不是在这个chunk 中
break;
}
pos += dpos; // pos 从头遍历累加
ce++; // 访问下一个entry
cr->chunk_pos++; // chunk_pos ++，记录游标当前在第几个chunk位置
}
if (ce->samples_per_chunk == 0) {
return NGX_ERROR;
}
dchunk = (cr->pos - pos) / ngx_rtmp_r32(ce->samples_per_chunk); // seek 之后的当前pos 位置，与找到chunk的pos差值，相同连续的chunk 可能不记录写进去。
// 比如: first_chunk 1,2,3,4,5,6,12,13
// samples_per_chunk 1,4,2,4,2,4,3, 4 ---> 1 + 4 + 2 + 4 + 2 + 4 = 17 , 18 --> 38 之间都是; 这个时候会有一个 dchunk差值
cr->chunk = ngx_rtmp_r32(ce->first_chunk) + dchunk; // 得到当前游标chunk的值，真实对应chunk 位置。
cr->chunk_pos = (ngx_uint_t) (ce - t->chunks->entries); // cr->chunk_pos 记录的是存储的真实解析位置
cr->chunk_count = (ngx_uint_t) (cr->pos - pos - dchunk *
ngx_rtmp_r32(ce->samples_per_chunk)); // 剩余是 chunk 偏移的变量帧
return ngx_rtmp_mp4_update_offset(s, t);
}
static ngx_int_t
ngx_rtmp_mp4_update_offset(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{
ngx_rtmp_mp4_cursor_t *cr;
ngx_uint_t chunk;
cr = &t->cursor;
if (cr->chunk < 1) {
return NGX_ERROR;
}
chunk = cr->chunk - 1; // 数组访问 -1
if (t->offsets) {
if (chunk >= ngx_rtmp_r32(t->offsets->entry_count)) {
return NGX_ERROR;
}
cr->offset = (off_t) ngx_rtmp_r32(t->offsets->entries[chunk]); // 取出第几个chunk偏移大小进行赋值
cr->size = 0;
return NGX_OK;
}
if (t->offsets64) {
if (chunk >= ngx_rtmp_r32(t->offsets64->entry_count)) {
return NGX_ERROR;
}
cr->offset = (off_t) ngx_rtmp_r64(t->offsets64->entries[chunk]);
cr->size = 0;
return NGX_OK;
}
return NGX_ERROR;
}ngx_rtmp_mp4_seek_size
|------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|
| stsz typedef struct { uint32_t version_flags; 4 字节 uint32_t sample_size; uint32_t sample_count; uint32_t entries[0]; } ngx_rtmp_mp4_sizes_t; | 
|
static ngx_int_t
ngx_rtmp_mp4_seek_size(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{
ngx_rtmp_mp4_cursor_t *cr;
ngx_uint_t pos;
cr = &t->cursor; // 当前游标
if (cr->chunk_count > cr->pos) { // 》 ？？
return NGX_ERROR;
}
if (t->sizes) {
if (t->sizes->sample_size) {
cr->size = ngx_rtmp_r32(t->sizes->sample_size);
cr->offset += cr->size * cr->chunk_count;
return NGX_OK;
}
if (cr->pos >= ngx_rtmp_r32(t->sizes->sample_count)) {
return NGX_ERROR;
}
for (pos = 1; pos <= cr->chunk_count; ++pos) { // chunk_count ??
cr->offset += ngx_rtmp_r32(t->sizes->entries[cr->pos - pos]);
}
cr->size_pos = cr->pos; // seek 时间戳当前pos
cr->size = ngx_rtmp_r32(t->sizes->entries[cr->size_pos]); // 获取当前pos对应帧数据大小
return NGX_OK;
}
if (t->sizes2) {
if (cr->size_pos >= ngx_rtmp_r32(t->sizes2->sample_count)) {
return NGX_ERROR;
}
cr->size_pos = cr->pos;
return NGX_OK;
}
return NGX_ERROR;
}ngx_rtmp_mp4_seek_delay
|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|
| ctts typedef struct { uint32_t sample_count; uint32_t sample_offset; } ngx_rtmp_mp4_delay_entry_t; typedef struct { uint32_t version_flags; uint32_t entry_count; ngx_rtmp_mp4_delay_entry_t entries[0]; } ngx_rtmp_mp4_delays_t; | 
|
static ngx_int_t
ngx_rtmp_mp4_seek_delay(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{
ngx_rtmp_mp4_cursor_t *cr;
ngx_rtmp_mp4_delay_entry_t *de;
uint32_t pos, dpos;
cr = &t->cursor;
if (t->delays == NULL) {
return NGX_OK;
}
pos = 0;
de = t->delays->entries;
while (cr->delay_pos < ngx_rtmp_r32(t->delays->entry_count)) { // 遍历所有entry
dpos = ngx_rtmp_r32(de->sample_count); // 2 表示：sample的个数
if (pos + dpos > cr->pos) { // 0 + 2 > 0 , 当前pos 帧在其中
cr->delay_count = cr->pos - pos; // 差值 count
cr->delay = ngx_rtmp_r32(de->sample_offset); // 偏移大小
break;
}
cr->delay_pos++; // 记录entry访问的位置
pos += dpos; // pos 累加计数
de++; // 下一个entry
}
if (cr->delay_pos >= ngx_rtmp_r32(t->delays->entry_count)) {
return NGX_OK;
}
return NGX_OK;
}3> ngx_rtmp_mp4_next play 播放一直取下一帧数据
static ngx_int_t
ngx_rtmp_mp4_next(ngx_rtmp_session_t *s, ngx_rtmp_mp4_track_t *t)
{
if (ngx_rtmp_mp4_next_time(s, t) != NGX_OK ||
ngx_rtmp_mp4_next_key(s, t) != NGX_OK ||
ngx_rtmp_mp4_next_chunk(s, t) != NGX_OK ||
ngx_rtmp_mp4_next_size(s, t) != NGX_OK ||
ngx_rtmp_mp4_next_delay(s, t) != NGX_OK)
{
t->cursor.valid = 0;
return NGX_ERROR;
}
t->cursor.valid = 1;
return NGX_OK;
}六：代码调试
使用进行查看变量的值： ngx_rtmp_r32(t->delays->entry_count)