一、引言
从《音视频入门基础:FLV专题(15)------Video Tag简介》可以知道,未加密的情况下,FLV文件中的一个Video Tag = Tag header + VideoTagHeader + VIDEODATA。本文讲述FFmpeg源码中是怎样解码Video Tag的VideoTagHeader,拿到里面的信息。
二、flv_read_packet函数
从《音视频入门基础:FLV专题(8)------FFmpeg源码中,解码Tag header的实现》可以知道,
FFmpeg源码中使用flv_read_packet函数来读取每个Tag的信息,该函数的前半部分实现了解码Tag header,获取其TagType属性的功能。然后根据TagType属性的值,判断该Tag为音频Tag、视频Tag还是脚本Tag。根据Tag的类型分别执行不同的解码操作:
cpp
if (type == FLV_TAG_TYPE_AUDIO) {
//...
} else if (type == FLV_TAG_TYPE_VIDEO) {
//...
}else if (type == FLV_TAG_TYPE_META) {
//...
}else{
//...
}
//...
如果在flv_read_packet函数的前半部分判断出该Tag为Video Tag,flv_read_packet函数中会执行如下逻辑解码Video Tag的VideoTagHeader:
cpp
else if (type == FLV_TAG_TYPE_VIDEO) {
stream_type = FLV_STREAM_TYPE_VIDEO;
flags = avio_r8(s->pb);
video_codec_id = flags & FLV_VIDEO_CODECID_MASK;
/*
* Reference Enhancing FLV 2023-03-v1.0.0-B.8
* https://github.com/veovera/enhanced-rtmp/blob/main/enhanced-rtmp-v1.pdf
* */
enhanced_flv = (flags >> 7) & 1;
size--;
if (enhanced_flv) {
video_codec_id = avio_rb32(s->pb);
size -= 4;
}
if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
int pkt_type = flags & 0x0F;
if (pkt_type == PacketTypeMetadata) {
int ret = flv_parse_video_color_info(s, st, next);
av_log(s, AV_LOG_DEBUG, "enhanced flv parse metadata ret %d and skip\n", ret);
}
goto skip;
} else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
goto skip;
}
}
//...
if (stream_type == FLV_STREAM_TYPE_AUDIO) {
//...
}else if (stream_type == FLV_STREAM_TYPE_VIDEO) {
int ret = flv_set_video_codec(s, st, video_codec_id, 1);
if (ret < 0)
return ret;
size -= ret;
}
//...
if (st->codecpar->codec_id == AV_CODEC_ID_AAC ||
st->codecpar->codec_id == AV_CODEC_ID_H264 ||
st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
st->codecpar->codec_id == AV_CODEC_ID_HEVC ||
st->codecpar->codec_id == AV_CODEC_ID_AV1 ||
st->codecpar->codec_id == AV_CODEC_ID_VP9) {
int type = 0;
if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO) {
type = flags & 0x0F;
} else {
type = avio_r8(s->pb);
size--;
}
if (size < 0) {
ret = AVERROR_INVALIDDATA;
goto leave;
}
if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && flv->meta_color_info_flag) {
flv_update_video_color_info(s, st); // update av packet side data
flv->meta_color_info_flag = 0;
}
if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
(st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {
// sign extension
int32_t cts = (avio_rb24(s->pb) + 0xff800000) ^ 0xff800000;
pts = av_sat_add64(dts, cts);
if (cts < 0) { // dts might be wrong
if (!flv->wrong_dts)
av_log(s, AV_LOG_WARNING,
"Negative cts, previous timestamps might be wrong.\n");
flv->wrong_dts = 1;
} else if (FFABS(dts - pts) > 1000*60*15) {
av_log(s, AV_LOG_WARNING,
"invalid timestamps %"PRId64" %"PRId64"\n", dts, pts);
dts = pts = AV_NOPTS_VALUE;
}
size -= 3;
}
//...
}
}
//...
下面我们分析上述代码块中解码Video Tag的VideoTagHeader的原理。
三、flv_read_packet函数中解码Video Tag的VideoTagHeader的原理
上述代码块中,首先通过avio_r8函数获取VideoTagHeader的第一个字节,也就是Frame Type(占4位) + CodecID(占4位),存贮到局部变量flags中。关于avio_r8函数的用法可以参考:《FFmpeg源码:avio_r8、avio_rl16、avio_rl24、avio_rl32、avio_rl64函数分析》:
cpp
flags = avio_r8(s->pb);
宏FLV_VIDEO_CODECID_MASK定义在libavformat/flv.h中:
cpp
#define FLV_VIDEO_CODECID_MASK 0x0f
通过下面语句将VideoTagHeader的CodecID属性提取出来,存贮到局部变量video_codec_id中:
cpp
video_codec_id = flags & FLV_VIDEO_CODECID_MASK;
下面这部分代码是用来判断文件格式是不是Enhancing FLV的:
cpp
/*
* Reference Enhancing FLV 2023-03-v1.0.0-B.8
* https://github.com/veovera/enhanced-rtmp/blob/main/enhanced-rtmp-v1.pdf
* */
enhanced_flv = (flags >> 7) & 1;
size--;
if (enhanced_flv) {
video_codec_id = avio_rb32(s->pb);
size -= 4;
}
if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
int pkt_type = flags & 0x0F;
if (pkt_type == PacketTypeMetadata) {
int ret = flv_parse_video_color_info(s, st, next);
av_log(s, AV_LOG_DEBUG, "enhanced flv parse metadata ret %d and skip\n", ret);
}
goto skip;
}
通过VideoTagHeader的Frame Type属性的值来判断该帧是不是视频信息/命令帧。如果是,执行语句:goto skip,不再继续解码VideoTagHeader:
cpp
else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
goto skip;
}
由《音视频入门基础:FLV专题(15)------Video Tag简介》可以知道,VideoTagHeader的CodecID属性为编解码器的标识符,表示该Video Tag的视频数据使用的是哪种视频压缩编码方式。通过语句flv_set_video_codec(s, st, video_codec_id, 1)设置st->codecpar->codec_id为CodecID属性对应的视频压缩编码方式:
cpp
else if (stream_type == FLV_STREAM_TYPE_VIDEO) {
int ret = flv_set_video_codec(s, st, video_codec_id, 1);
if (ret < 0)
return ret;
size -= ret;
}
如果文件格式不是Enhancing FLV,通过语句:type = avio_r8(s->pb)读取VideoTagHeader的AVCPacketType属性:
cpp
if (st->codecpar->codec_id == AV_CODEC_ID_AAC ||
st->codecpar->codec_id == AV_CODEC_ID_H264 ||
st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
st->codecpar->codec_id == AV_CODEC_ID_HEVC ||
st->codecpar->codec_id == AV_CODEC_ID_AV1 ||
st->codecpar->codec_id == AV_CODEC_ID_VP9) {
int type = 0;
if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO) {
type = flags & 0x0F;
} else {
type = avio_r8(s->pb);
size--;
}
//...
读取VideoTagHeader的CompositionTime属性,赋值给局部变量cts。通过语句:pts = av_sat_add64(dts, cts),让pts = dts + cts,从而得到pts的值。dts来源于FLV文件的Tag header的Timestamp和TimestampExtended属性,具体可以参考:《音视频入门基础:FLV专题(7)------Tag header简介》和《音视频入门基础:FLV专题(8)------FFmpeg源码中,解码Tag header的实现》。关于av_sat_add64函数的用法可以参考:《FFmpeg源码:av_sat_add64_c、av_sat_sub64_c函数分析》:
cpp
if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
(st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {
// sign extension
int32_t cts = (avio_rb24(s->pb) + 0xff800000) ^ 0xff800000;
pts = av_sat_add64(dts, cts);
if (cts < 0) { // dts might be wrong
if (!flv->wrong_dts)
av_log(s, AV_LOG_WARNING,
"Negative cts, previous timestamps might be wrong.\n");
flv->wrong_dts = 1;
} else if (FFABS(dts - pts) > 1000*60*15) {
av_log(s, AV_LOG_WARNING,
"invalid timestamps %"PRId64" %"PRId64"\n", dts, pts);
dts = pts = AV_NOPTS_VALUE;
}
size -= 3;
}