本文档结合FFmpeg源代码，详细解释RTSP拉流时如何接收和解析RTP包，还原成完整媒体帧的全过程。
一、整体流程概览
网络接收 (UDP/TCP)
│
▼
rtsp_read_packet() [libavformat/rtspdec.c:848]
│
├─→ 处理Real Server订阅逻辑
│
▼
ff_rtsp_fetch_packet() [libavformat/rtsp.c:2266]
│
├─→ 检查是否有缓存的帧可以直接返回
├─→ 检查重排序队列
│
▼
read_packet() [内部函数]
│
├─→ UDP模式: 从UDP socket读取
├─→ TCP模式: ff_rtsp_tcp_read_packet()
│
▼
ff_rtp_parse_packet() [libavformat/rtpdec.c:926]
│
├─→ SRTP解密 (如果启用)
├─→ rtp_parse_one_packet()
│
▼
rtp_parse_packet_internal() [libavformat/rtpdec.c:679]
│
├─→ 解析RTP固定头 (12字节)
├─→ 序列号验证
├─→ 处理Padding和CSRC
├─→ 处理RTP扩展头
│
▼
编解码器特定解包函数 (handler->parse_packet)
├─→ H.264: h264_handle_packet()
├─→ AAC: aac_parse_packet()
├─→ 其他编解码器...
│
▼
finalize_packet() [libavformat/rtpdec.c:631]
│
├─→ 时间戳计算和转换
├─→ 添加RTCP SR侧数据
│
▼
AVPacket (完整的媒体帧)二、核心函数详解
2.1 rtsp_read_packet() - RTSP拉流入口
位置 : libavformat/rtspdec.c:848
c
static int rtsp_read_packet(AVFormatContext *s, AVPacket *pkt)
{
RTSPState *rt = s->priv_data;
int ret;
RTSPMessageHeader reply1, *reply = &reply1;
char cmd[MAX_URL_SIZE];
retry:
// 1. Real Server特殊处理 (订阅/取消订阅)
if (rt->server_type == RTSP_SERVER_REAL) {
// ... Real Server订阅逻辑
}
// 2. 调用核心获取函数
ret = ff_rtsp_fetch_packet(s, pkt);
// 3. UDP超时处理 - 自动切换到TCP
if (ret < 0) {
if (ret == AVERROR(ETIMEDOUT) && !rt->packets) {
if (rt->lower_transport == RTSP_LOWER_TRANSPORT_UDP &&
rt->lower_transport_mask & (1 << RTSP_LOWER_TRANSPORT_TCP)) {
av_log(s, AV_LOG_WARNING, "UDP timeout, retrying with TCP\n");
// 重新建立TCP连接
if (resetup_tcp(s) == 0) {
rt->state = RTSP_STATE_IDLE;
rt->need_subscription = 1;
if (rtsp_read_play(s) != 0)
return -1;
goto retry;
}
}
}
return ret;
}
rt->packets++;
// 4. 发送保活消息 (每timeout/2秒)
if (!(rt->rtsp_flags & RTSP_FLAG_LISTEN)) {
if ((av_gettime_relative() - rt->last_cmd_time) / 1000000 >= rt->timeout / 2) {
if (rt->server_type == RTSP_SERVER_WMS ||
(rt->server_type != RTSP_SERVER_REAL &&
rt->get_parameter_supported)) {
ff_rtsp_send_cmd_async(s, "GET_PARAMETER", rt->control_uri, NULL);
} else {
ff_rtsp_send_cmd_async(s, "OPTIONS", rt->control_uri, NULL);
}
}
}
return 0;
}关键点:
- 处理Real Server的订阅机制
- UDP超时自动切换到TCP
- 定期发送保活消息 (GET_PARAMETER或OPTIONS)
2.2 ff_rtsp_fetch_packet() - 核心获取函数
位置 : libavformat/rtsp.c:2266
c
int ff_rtsp_fetch_packet(AVFormatContext *s, AVPacket *pkt)
{
RTSPState *rt = s->priv_data;
int ret, len;
RTSPStream *rtsp_st, *first_queue_st = NULL;
int64_t wait_end = 0;
// 1. 检查是否所有流都收到BYE (结束信号)
if (rt->nb_byes == rt->nb_rtsp_streams)
return AVERROR_EOF;
// 2. 检查是否有缓存的帧可以直接返回
if (rt->cur_transport_priv) {
if (rt->transport == RTSP_TRANSPORT_RTP) {
ret = ff_rtp_parse_packet(rt->cur_transport_priv, pkt, NULL, 0);
}
if (ret == 0) {
rt->cur_transport_priv = NULL;
return 0; // 成功返回一个完整帧
} else if (ret == 1) {
return 0; // 返回了一个帧，还有更多帧可用
} else
rt->cur_transport_priv = NULL;
}
redo:
// 3. 检查重排序队列，找到最早入队的包
if (rt->transport == RTSP_TRANSPORT_RTP) {
int i;
int64_t first_queue_time = 0;
for (i = 0; i < rt->nb_rtsp_streams; i++) {
RTPDemuxContext *rtpctx = rt->rtsp_streams[i]->transport_priv;
int64_t queue_time;
if (!rtpctx)
continue;
queue_time = ff_rtp_queued_packet_time(rtpctx);
if (queue_time && (queue_time - first_queue_time < 0 ||
!first_queue_time)) {
first_queue_time = queue_time;
first_queue_st = rt->rtsp_streams[i];
}
}
if (first_queue_time) {
wait_end = first_queue_time + s->max_delay;
}
}
// 4. 分配接收缓冲区
if (!rt->recvbuf) {
rt->recvbuf = av_malloc(RECVBUF_SIZE);
if (!rt->recvbuf)
return AVERROR(ENOMEM);
}
// 5. 读取下一个RTP包
len = read_packet(s, &rtsp_st, first_queue_st, wait_end);
// 6. 如果超时且有队列中的包，直接处理队列中的包
if (len == AVERROR(EAGAIN) && first_queue_st &&
rt->transport == RTSP_TRANSPORT_RTP) {
av_log(s, AV_LOG_WARNING,
"max delay reached. need to consume packet\n");
rtsp_st = first_queue_st;
ret = ff_rtp_parse_packet(rtsp_st->transport_priv, pkt, NULL, 0);
goto end;
}
if (len < 0)
return len;
// 7. 解析RTP包
if (rt->transport == RTSP_TRANSPORT_RTP) {
ret = ff_rtp_parse_packet(rtsp_st->transport_priv, pkt, &rt->recvbuf, len);
// 8. 发送RTCP反馈 (NACK/PLI)
if (rtsp_st->feedback) {
ff_rtp_send_rtcp_feedback(rtsp_st->transport_priv, rtsp_st->rtp_handle, NULL);
}
// 9. 同步RTCP时间戳
if (ret < 0) {
RTPDemuxContext *rtpctx = rtsp_st->transport_priv;
if (rtpctx->first_rtcp_ntp_time != AV_NOPTS_VALUE) {
// 将RTCP NTP时间同步到所有流
for (i = 0; i < rt->nb_rtsp_streams; i++) {
RTPDemuxContext *rtpctx2 = rt->rtsp_streams[i]->transport_priv;
if (rtpctx2 && rtpctx2->first_rtcp_ntp_time == AV_NOPTS_VALUE) {
rtpctx2->first_rtcp_ntp_time = rtpctx->first_rtcp_ntp_time;
rtpctx2->rtcp_ts_offset = av_rescale_q(
rtpctx->rtcp_ts_offset, st->time_base, st2->time_base);
}
}
}
if (ret == -RTCP_BYE) {
rt->nb_byes++;
if (rt->nb_byes == rt->nb_rtsp_streams)
return AVERROR_EOF;
}
}
}
end:
if (ret < 0)
goto redo;
if (ret == 1)
// 还有更多帧可用，保存RTP上下文
rt->cur_transport_priv = rtsp_st->transport_priv;
return ret;
}关键点:
- 缓存机制：避免重复解析
- 重排序队列：处理乱序到达的包
- 超时处理：max_delay机制
- RTCP反馈：NACK/PLI支持
- 时间戳同步：多流同步
2.3 TCP Interleaved模式读取
位置 : libavformat/rtspdec.c:785
c
int ff_rtsp_tcp_read_packet(AVFormatContext *s, RTSPStream **prtsp_st,
uint8_t *buf, int buf_size)
{
RTSPState *rt = s->priv_data;
int id, len, i, ret;
RTSPStream *rtsp_st;
redo:
// 1. 循环读取，直到遇到'$'标记 (Interleaved包)
for (;;) {
RTSPMessageHeader reply;
ret = ff_rtsp_read_reply(s, &reply, NULL, 1, NULL);
if (ret < 0)
return ret;
if (ret == 1) /* received '$' */
break;
// 处理RTSP控制消息
if (rt->state != RTSP_STATE_STREAMING)
return 0;
}
// 2. 读取Interleaved头 (3字节)
ret = ffurl_read_complete(rt->rtsp_hd, buf, 3);
if (ret != 3)
return AVERROR(EIO);
id = buf[0]; // Channel ID
len = AV_RB16(buf + 1); // Packet Length
av_log(s, AV_LOG_TRACE, "id=%d len=%d\n", id, len);
// 3. 验证包大小
if (len > buf_size || len < 8)
goto redo;
// 4. 读取RTP/RTCP包数据
ret = ffurl_read_complete(rt->rtsp_hd, buf, len);
if (ret != len)
return AVERROR(EIO);
// 5. 根据Channel ID找到对应的RTSPStream
for (i = 0; i < rt->nb_rtsp_streams; i++) {
rtsp_st = rt->rtsp_streams[i];
if (id >= rtsp_st->interleaved_min &&
id <= rtsp_st->interleaved_max)
goto found;
}
goto redo;
found:
*prtsp_st = rtsp_st;
return len;
}TCP Interleaved格式:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| '$' (0x24) | Channel ID | Packet Length (16) |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| RTP/RTCP Packet Data |
| ... |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+2.4 ff_rtp_parse_packet() - RTP包解析入口
位置 : libavformat/rtpdec.c:926
c
int ff_rtp_parse_packet(RTPDemuxContext *s, AVPacket *pkt,
uint8_t **bufptr, int len)
{
int rv;
// 1. SRTP解密 (如果启用)
if (s->srtp_enabled && bufptr && ff_srtp_decrypt(&s->srtp, *bufptr, &len) < 0)
return -1;
// 2. 解析单个RTP包
rv = rtp_parse_one_packet(s, pkt, bufptr, len);
s->prev_ret = rv;
// 3. 如果解析失败，尝试从队列中取包
while (rv < 0 && has_next_packet(s))
rv = rtp_parse_queued_packet(s, pkt);
// 返回值:
// 0: 成功返回一个完整包，没有更多包
// 1: 成功返回一个包，还有更多包可用
// -1: 没有包可用
return rv ? rv : has_next_packet(s);
}2.5 rtp_parse_one_packet() - 单包解析
位置 : libavformat/rtpdec.c:839
c
static int rtp_parse_one_packet(RTPDemuxContext *s, AVPacket *pkt,
uint8_t **bufptr, int len)
{
uint8_t *buf = bufptr ? *bufptr : NULL;
int flags = 0;
uint32_t timestamp;
int rv = 0;
// 1. 如果buf为NULL，从队列中取包
if (!buf) {
if (s->prev_ret <= 0)
return rtp_parse_queued_packet(s, pkt);
// 调用handler继续解析
if (s->handler && s->handler->parse_packet) {
timestamp = RTP_NOTS_VALUE;
rv = s->handler->parse_packet(s->ic, s->dynamic_protocol_context,
s->st, pkt, ×tamp, NULL, 0, 0, flags);
finalize_packet(s, pkt, timestamp);
return rv;
}
}
// 2. 验证包大小
if (len < 12)
return -1;
// 3. 验证RTP版本
if ((buf[0] & 0xc0) != (RTP_VERSION << 6))
return -1;
// 4. 检查是否是RTCP包
if (RTP_PT_IS_RTCP(buf[1])) {
return rtcp_parse_packet(s, buf, len);
}
// 5. 计算Jitter (抖动)
if (s->st) {
int64_t received = av_gettime_relative();
uint32_t arrival_ts = av_rescale_q(received, AV_TIME_BASE_Q,
s->st->time_base);
timestamp = AV_RB32(buf + 4);
rtcp_update_jitter(&s->statistics, timestamp, arrival_ts);
}
// 6. 处理重排序
if ((s->seq == 0 && !s->queue) || s->queue_size <= 1) {
// 第一个包或禁用重排序，直接解析
return rtp_parse_packet_internal(s, pkt, buf, len);
} else {
uint16_t seq = AV_RB16(buf + 2);
int16_t diff = seq - s->seq;
if (diff < 0) {
// 旧包，丢弃
av_log(s->ic, AV_LOG_WARNING,
"RTP: dropping old packet received too late\n");
return -1;
} else if (diff <= 1) {
// 正确的包，直接解析
rv = rtp_parse_packet_internal(s, pkt, buf, len);
return rv;
} else {
// 乱序包，入队
rv = enqueue_packet(s, buf, len);
if (rv < 0)
return rv;
*bufptr = NULL;
// 如果队列满了，强制输出最早的包
if (s->queue_len >= s->queue_size) {
av_log(s->ic, AV_LOG_WARNING, "jitter buffer full\n");
return rtp_parse_queued_packet(s, pkt);
}
return -1;
}
}
}关键点:
- RTCP包识别和处理
- Jitter计算
- 重排序队列管理
- 乱序包处理
2.6 rtp_parse_packet_internal() - RTP头解析
位置 : libavformat/rtpdec.c:679
c
static int rtp_parse_packet_internal(RTPDemuxContext *s, AVPacket *pkt,
const uint8_t *buf, int len)
{
unsigned int ssrc;
int payload_type, seq, flags = 0;
int ext, csrc;
AVStream *st;
uint32_t timestamp;
int rv = 0;
// 1. 解析RTP固定头 (12字节)
csrc = buf[0] & 0x0f; // CSRC计数
ext = buf[0] & 0x10; // Extension标志
payload_type = buf[1] & 0x7f; // Payload Type
if (buf[1] & 0x80)
flags |= RTP_FLAG_MARKER; // Marker位
seq = AV_RB16(buf + 2); // 序列号
timestamp = AV_RB32(buf + 4); // 时间戳
ssrc = AV_RB32(buf + 8); // SSRC
s->ssrc = ssrc;
// 2. 验证Payload Type
if (s->payload_type != payload_type)
return -1;
st = s->st;
// 3. 序列号验证
if (!rtp_valid_packet_in_sequence(&s->statistics, seq)) {
av_log(s->ic, AV_LOG_ERROR,
"RTP: PT=%02x: bad cseq %04x expected=%04x\n",
payload_type, seq, ((s->seq + 1) & 0xffff));
return -1;
}
// 4. 处理Padding
if (buf[0] & 0x20) {
int padding = buf[len - 1];
if (len >= 12 + padding)
len -= padding;
}
s->seq = seq;
len -= 12;
buf += 12;
// 5. 跳过CSRC列表
len -= 4 * csrc;
buf += 4 * csrc;
if (len < 0)
return AVERROR_INVALIDDATA;
// 6. 处理RTP扩展头 (RFC 3550 Section 5.3.1)
if (ext) {
if (len < 4)
return -1;
// 扩展头长度 (32位字数)
ext = (AV_RB16(buf + 2) + 1) << 2;
if (len < ext)
return -1;
// 跳过扩展头
len -= ext;
buf += ext;
}
// 7. 调用编解码器特定的解包函数
if (s->handler && s->handler->parse_packet) {
rv = s->handler->parse_packet(s->ic, s->dynamic_protocol_context,
s->st, pkt, ×tamp, buf, len, seq,
flags);
} else if (st) {
// 默认处理：直接复制数据
if ((rv = av_new_packet(pkt, len)) < 0)
return rv;
memcpy(pkt->data, buf, len);
pkt->stream_index = st->index;
} else {
return AVERROR(EINVAL);
}
// 8. 时间戳处理
finalize_packet(s, pkt, timestamp);
return rv;
}RTP固定头结构 (12字节):
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|V=2|P|X| CC |M| PT | Sequence Number |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Timestamp |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| Synchronization Source (SSRC) identifier |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+2.7 序列号验证
位置 : libavformat/rtpdec.c:249
c
static int rtp_valid_packet_in_sequence(RTPStatistics *s, uint16_t seq)
{
uint16_t udelta = seq - s->max_seq;
const int MAX_DROPOUT = 3000;
const int MAX_MISORDER = 100;
const int MIN_SEQUENTIAL = 2;
// 1. Probation期：需要连续收到MIN_SEQUENTIAL个包
if (s->probation) {
if (seq == s->max_seq + 1) {
s->probation--;
s->max_seq = seq;
if (s->probation == 0) {
rtp_init_sequence(s, seq);
s->received++;
return 1;
}
} else {
s->probation = MIN_SEQUENTIAL - 1;
s->max_seq = seq;
}
}
// 2. 正常序列，允许小间隙
else if (udelta < MAX_DROPOUT) {
// 序列号回绕
if (seq < s->max_seq) {
s->cycles += RTP_SEQ_MOD;
}
s->max_seq = seq;
}
// 3. 序列号大跳跃
else if (udelta <= RTP_SEQ_MOD - MAX_MISORDER) {
if (seq == s->bad_seq) {
// 连续两个包，认为是重启，重新同步
rtp_init_sequence(s, seq);
} else {
s->bad_seq = (seq + 1) & (RTP_SEQ_MOD - 1);
return 0;
}
}
// 4. 重复或乱序包
else {
// 不更新统计，但接受包
}
s->received++;
return 1;
}序列号验证逻辑:
- Probation期: 连续收到2个正确序列的包才开始接收
- 正常接收: 允许最多3000个包的间隙
- 重启检测: 大跳跃后需要连续两个包确认
- 乱序处理: 允许最多100个包的乱序
2.8 RTCP包解析
位置 : libavformat/rtpdec.c:181
c
static int rtcp_parse_packet(RTPDemuxContext *s, const unsigned char *buf,
int len)
{
int payload_len;
while (len >= 4) {
payload_len = FFMIN(len, (AV_RB16(buf + 2) + 1) * 4);
switch (buf[1]) {
case RTCP_SR: // Sender Report
if (payload_len < 28) {
av_log(s->ic, AV_LOG_ERROR, "Invalid RTCP SR packet length\n");
return AVERROR_INVALIDDATA;
}
// 解析SR字段
s->last_sr.ssrc = AV_RB32(buf + 4);
s->last_sr.ntp_timestamp = AV_RB64(buf + 8);
s->last_sr.rtp_timestamp = AV_RB32(buf + 16);
s->last_sr.sender_nb_packets = AV_RB32(buf + 20);
s->last_sr.sender_nb_bytes = AV_RB32(buf + 24);
s->pending_sr = 1;
s->last_rtcp_reception_time = av_gettime_relative();
// 初始化RTCP时间戳
if (s->first_rtcp_ntp_time == AV_NOPTS_VALUE) {
s->first_rtcp_ntp_time = s->last_sr.ntp_timestamp;
if (!s->base_timestamp)
s->base_timestamp = s->last_sr.rtp_timestamp;
s->rtcp_ts_offset = (int32_t)(s->last_sr.rtp_timestamp - s->base_timestamp);
}
break;
case RTCP_BYE: // Goodbye
return -RTCP_BYE;
}
buf += payload_len;
len -= payload_len;
}
return -1;
}RTCP SR (Sender Report) 格式:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
|V=2|P| RC | PT=SR=200 | length |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| SSRC of sender |
+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+
| NTP timestamp, most significant word |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| NTP timestamp, least significant word |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| RTP timestamp |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| sender's packet count |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| sender's octet count |
+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+用途:
- 时间同步: NTP时间戳用于音视频同步
- 统计信息: 包数和字节数用于QoS监控
- 时间戳映射: RTP时间戳到NTP时间的映射
2.9 finalize_packet() - 时间戳处理
位置 : libavformat/rtpdec.c:631
c
static void finalize_packet(RTPDemuxContext *s, AVPacket *pkt, uint32_t timestamp)
{
// 1. 添加RTCP SR侧数据
if (s->pending_sr) {
int ret = rtp_add_sr_sidedata(s, pkt);
if (ret < 0)
av_log(s->ic, AV_LOG_WARNING, "rtpdec: failed to add SR sidedata\n");
}
// 2. 如果depacketizer已经设置了时间戳，直接返回
if (pkt->pts != AV_NOPTS_VALUE || pkt->dts != AV_NOPTS_VALUE)
return;
if (timestamp == RTP_NOTS_VALUE)
return;
// 3. 添加Producer Reference Time侧数据
if (s->last_sr.ntp_timestamp != AV_NOPTS_VALUE) {
if (rtp_set_prft(s, pkt, timestamp) < 0) {
av_log(s->ic, AV_LOG_WARNING, "rtpdec: failed to set prft");
}
}
// 4. 基于RTCP SR计算PTS (多流同步)
if (s->last_sr.ntp_timestamp != AV_NOPTS_VALUE && s->ic->nb_streams > 1) {
int64_t addend;
int32_t delta_timestamp;
// 计算RTP时间戳差值
delta_timestamp = (int32_t)(timestamp - s->last_sr.rtp_timestamp);
// 转换到PTS时间基
addend = av_rescale(s->last_sr.ntp_timestamp - s->first_rtcp_ntp_time,
s->st->time_base.den,
(uint64_t) s->st->time_base.num << 32);
pkt->pts = s->range_start_offset + s->rtcp_ts_offset + addend +
delta_timestamp;
return;
}
// 5. 基于RTP时间戳计算PTS (单流)
if (!s->base_timestamp)
s->base_timestamp = timestamp;
// 处理时间戳回绕
if (!s->timestamp)
s->unwrapped_timestamp += timestamp;
else
s->unwrapped_timestamp += (int32_t)(timestamp - s->timestamp);
s->timestamp = timestamp;
pkt->pts = s->unwrapped_timestamp + s->range_start_offset -
s->base_timestamp;
}时间戳计算方式:
-
基于RTCP SR (多流同步):
- 使用NTP时间戳作为参考
- 计算RTP时间戳偏移
- 适用于音视频同步
-
基于RTP时间戳 (单流):
- 相对于base_timestamp计算
- 处理32位时间戳回绕
- 简单高效
三、编解码器特定解包详解
3.1 H.264解包 - FU-A重组
位置 : libavformat/rtpdec_h264.c:313
c
static int h264_handle_packet(AVFormatContext *ctx, PayloadContext *data,
AVStream *st, AVPacket *pkt, uint32_t *timestamp,
const uint8_t *buf, int len, uint16_t seq,
int flags)
{
uint8_t nal;
uint8_t type;
int result = 0;
if (!len) {
av_log(ctx, AV_LOG_ERROR, "Empty H.264 RTP packet\n");
return AVERROR_INVALIDDATA;
}
nal = buf[0];
type = nal & 0x1f;
// 简化NAL类型 (1-23都是Single NAL Unit)
if (type >= 1 && type <= 23)
type = 1;
switch (type) {
case 0: // undefined
case 1: // Single NAL Unit
// 分配包空间 (起始码 + NAL数据)
if ((result = av_new_packet(pkt, len + sizeof(start_sequence))) < 0)
return result;
// 添加起始码 0x00000001
memcpy(pkt->data, start_sequence, sizeof(start_sequence));
memcpy(pkt->data + sizeof(start_sequence), buf, len);
break;
case 24: // STAP-A (Single-Time Aggregation Packet)
// 跳过STAP-A NAL头
buf++;
len--;
result = ff_h264_handle_aggregated_packet(ctx, data, pkt, buf, len, 0,
NAL_COUNTERS, NAL_MASK);
break;
case 25: // STAP-B
case 26: // MTAP-16
case 27: // MTAP-24
case 29: // FU-B
avpriv_report_missing_feature(ctx, "RTP H.264 NAL unit type %d", type);
result = AVERROR_PATCHWELCOME;
break;
case 28: // FU-A (Fragmentation Unit)
result = h264_handle_packet_fu_a(ctx, data, pkt, buf, len,
NAL_COUNTERS, NAL_MASK);
break;
case 30: // undefined
case 31: // undefined
default:
av_log(ctx, AV_LOG_ERROR, "Undefined type (%d)\n", type);
result = AVERROR_INVALIDDATA;
break;
}
return result;
}3.1.1 Single NAL Unit处理
c
// 输入: RTP Payload
// [NAL Header | NAL Data]
// 输出: AVPacket
// [0x00 0x00 0x00 0x01 | NAL Header | NAL Data]3.1.2 STAP-A处理 (聚合包)
位置 : libavformat/rtpdec_h264.c:207
c
int ff_h264_handle_aggregated_packet(AVFormatContext *ctx, PayloadContext *data, AVPacket *pkt,
const uint8_t *buf, int len,
int skip_between, int *nal_counters,
int nal_mask)
{
int pass = 0;
int total_length = 0;
uint8_t *dst = NULL;
int ret;
// 两遍处理：第一遍计算总长度，第二遍复制数据
for (pass = 0; pass < 2; pass++) {
const uint8_t *src = buf;
int src_len = len;
while (src_len > 2) {
// 读取NAL单元长度 (2字节)
uint16_t nal_size = AV_RB16(src);
src += 2;
src_len -= 2;
if (nal_size <= src_len) {
if (pass == 0) {
// 第一遍：计算总长度
total_length += sizeof(start_sequence) + nal_size;
} else {
// 第二遍：复制数据
memcpy(dst, start_sequence, sizeof(start_sequence));
dst += sizeof(start_sequence);
memcpy(dst, src, nal_size);
dst += nal_size;
}
} else {
av_log(ctx, AV_LOG_ERROR,
"nal size exceeds length: %d %d\n", nal_size, src_len);
return AVERROR_INVALIDDATA;
}
src += nal_size + skip_between;
src_len -= nal_size + skip_between;
}
if (pass == 0) {
// 分配包空间
if ((ret = av_new_packet(pkt, total_length)) < 0)
return ret;
dst = pkt->data;
}
}
return 0;
}STAP-A格式:
输入 (RTP Payload):
[STAP-A Header]
[NAL Size 1 (2 bytes)] [NAL Unit 1]
[NAL Size 2 (2 bytes)] [NAL Unit 2]
...
输出 (AVPacket):
[0x00 0x00 0x00 0x01] [NAL Unit 1]
[0x00 0x00 0x00 0x01] [NAL Unit 2]
...3.1.3 FU-A处理 (分片重组)
位置 : libavformat/rtpdec_h264.c:286
c
static int h264_handle_packet_fu_a(AVFormatContext *ctx, PayloadContext *data, AVPacket *pkt,
const uint8_t *buf, int len,
int *nal_counters, int nal_mask)
{
uint8_t fu_indicator, fu_header, start_bit, nal_type, nal;
if (len < 3) {
av_log(ctx, AV_LOG_ERROR, "Too short data for FU-A H.264 RTP packet\n");
return AVERROR_INVALIDDATA;
}
// 解析FU-A头
fu_indicator = buf[0];
fu_header = buf[1];
start_bit = fu_header >> 7; // S位
nal_type = fu_header & 0x1f; // NAL类型
nal = fu_indicator & 0xe0 | nal_type; // 重构NAL头
// 跳过FU Indicator和FU Header
buf += 2;
len -= 2;
if (start_bit && nal_counters)
nal_counters[nal_type & nal_mask]++;
return ff_h264_handle_frag_packet(pkt, buf, len, start_bit, &nal, 1);
}FU-A格式:
输入 (RTP Payload):
[FU Indicator] [FU Header] [Fragment Data]
FU Indicator (1 byte):
F (1 bit) | NRI (2 bits) | Type=28 (5 bits)
FU Header (1 byte):
S (1 bit) | E (1 bit) | R (1 bit) | NAL Type (5 bits)
输出 (AVPacket):
- 第一个分片 (S=1):
[0x00 0x00 0x00 0x01] [重构的NAL Header] [Fragment Data]
- 后续分片 (S=0):
[Fragment Data] (追加到现有包)3.2 AAC解包 - MPEG4-GENERIC
位置 : libavformat/rtpdec_mpeg4.c:181
c
static int aac_parse_packet(AVFormatContext *ctx, PayloadContext *data,
AVStream *st, AVPacket *pkt, uint32_t *timestamp,
const uint8_t *buf, int len, uint16_t seq,
int flags)
{
int ret;
// 1. 如果buf为NULL，返回缓存的AU
if (!buf) {
if (data->cur_au_index > data->nb_au_headers) {
av_log(ctx, AV_LOG_ERROR, "Invalid parser state\n");
return AVERROR_INVALIDDATA;
}
if (data->buf_size - data->buf_pos < data->au_headers[data->cur_au_index].size) {
av_log(ctx, AV_LOG_ERROR, "Invalid AU size\n");
return AVERROR_INVALIDDATA;
}
// 创建包并复制AU数据
if ((ret = av_new_packet(pkt, data->au_headers[data->cur_au_index].size)) < 0) {
return ret;
}
memcpy(pkt->data, &data->buf[data->buf_pos], data->au_headers[data->cur_au_index].size);
data->buf_pos += data->au_headers[data->cur_au_index].size;
pkt->stream_index = st->index;
data->cur_au_index++;
// 检查是否还有更多AU
if (data->cur_au_index == data->nb_au_headers) {
data->buf_pos = 0;
return 0; // 没有更多AU
}
return 1; // 还有更多AU
}
// 2. 解析AU Headers
if (rtp_parse_mp4_au(data, buf, len)) {
av_log(ctx, AV_LOG_ERROR, "Error parsing AU headers\n");
return -1;
}
buf += data->au_headers_length_bytes + 2;
len -= data->au_headers_length_bytes + 2;
// 3. 处理分片的AU
if (data->nb_au_headers == 1 && len < data->au_headers[0].size) {
// AU被分片了
if (!data->buf_pos) {
if (data->au_headers[0].size > MAX_AAC_HBR_FRAME_SIZE) {
av_log(ctx, AV_LOG_ERROR, "Invalid AU size\n");
return AVERROR_INVALIDDATA;
}
data->buf_size = data->au_headers[0].size;
data->timestamp = *timestamp;
}
// 验证时间戳和大小一致性
if (data->timestamp != *timestamp ||
data->au_headers[0].size != data->buf_size ||
data->buf_pos + len > MAX_AAC_HBR_FRAME_SIZE) {
data->buf_pos = 0;
data->buf_size = 0;
av_log(ctx, AV_LOG_ERROR, "Invalid packet received\n");
return AVERROR_INVALIDDATA;
}
// 复制分片数据
memcpy(&data->buf[data->buf_pos], buf, len);
data->buf_pos += len;
// 等待最后一个分片 (Marker位)
if (!(flags & RTP_FLAG_MARKER))
return AVERROR(EAGAIN);
// 验证完整性
if (data->buf_pos != data->buf_size) {
data->buf_pos = 0;
av_log(ctx, AV_LOG_ERROR, "Missed some packets, discarding frame\n");
return AVERROR_INVALIDDATA;
}
// 创建完整的包
data->buf_pos = 0;
ret = av_new_packet(pkt, data->buf_size);
if (ret < 0)
return ret;
pkt->stream_index = st->index;
memcpy(pkt->data, data->buf, data->buf_size);
return 0;
}
// 4. 完整的AU (未分片)
if (len < data->au_headers[0].size) {
av_log(ctx, AV_LOG_ERROR, "First AU larger than packet size\n");
return AVERROR_INVALIDDATA;
}
// 创建包并复制第一个AU
if ((ret = av_new_packet(pkt, data->au_headers[0].size)) < 0)
return ret;
memcpy(pkt->data, buf, data->au_headers[0].size);
len -= data->au_headers[0].size;
buf += data->au_headers[0].size;
pkt->stream_index = st->index;
// 5. 如果有多个AU，缓存剩余的
if (len > 0 && data->nb_au_headers > 1) {
data->buf_size = FFMIN(len, sizeof(data->buf));
memcpy(data->buf, buf, data->buf_size);
data->cur_au_index = 1;
data->buf_pos = 0;
return 1; // 还有更多AU
}
return 0;
}3.2.1 AU Header解析
位置 : libavformat/rtpdec_mpeg4.c:142
c
static int rtp_parse_mp4_au(PayloadContext *data, const uint8_t *buf, int len)
{
int au_headers_length, au_header_size, i;
GetBitContext getbitcontext;
if (len < 2)
return AVERROR_INVALIDDATA;
// 读取AU Headers Length (16 bits, 单位: bits)
au_headers_length = AV_RB16(buf);
if (au_headers_length > RTP_MAX_PACKET_LENGTH)
return -1;
data->au_headers_length_bytes = (au_headers_length + 7) / 8;
// AU Header大小 = sizelength + indexlength
au_header_size = data->sizelength + data->indexlength;
if (au_header_size <= 0 || (au_headers_length % au_header_size != 0))
return -1;
// 计算AU数量
data->nb_au_headers = au_headers_length / au_header_size;
if (!data->au_headers || data->au_headers_allocated < data->nb_au_headers) {
av_free(data->au_headers);
data->au_headers = av_malloc(sizeof(struct AUHeaders) * data->nb_au_headers);
if (!data->au_headers)
return AVERROR(ENOMEM);
data->au_headers_allocated = data->nb_au_headers;
}
init_get_bits(&getbitcontext, &buf[2], data->au_headers_length_bytes * 8);
// 解析每个AU Header
for (i = 0; i < data->nb_au_headers; ++i) {
data->au_headers[i].size = get_bits_long(&getbitcontext, data->sizelength);
data->au_headers[i].index = get_bits_long(&getbitcontext, data->indexlength);
}
return 0;
}AAC RTP Payload格式:
0 1 2 3
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| AU-headers-length | AU-header(1) |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| AU-header(2) | ... |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
| AU-data (Access Unit 1) |
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
| |
| AU-data (Access Unit 2) |
| |
+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+AU Header结构 (通常16 bits):
- AU-size (13 bits): AU大小 (单位: bytes)
- AU-Index (3 bits): AU索引 (通常为0)
四、重排序队列机制
4.1 入队操作
位置 : libavformat/rtpdec.c:775
c
static int enqueue_packet(RTPDemuxContext *s, uint8_t *buf, int len)
{
uint16_t seq = AV_RB16(buf + 2);
RTPPacket **cur = &s->queue, *packet;
// 1. 找到正确的插入位置 (按序列号排序)
while (*cur) {
int16_t diff = seq - (*cur)->seq;
if (diff < 0)
break;
cur = &(*cur)->next;
}
// 2. 创建新的RTPPacket节点
packet = av_mallocz(sizeof(*packet));
if (!packet)
return AVERROR(ENOMEM);
packet->recvtime = av_gettime_relative();
packet->seq = seq;
packet->len = len;
packet->buf = buf;
packet->next = *cur;
*cur = packet;
s->queue_len++;
return 0;
}4.2 出队操作
位置 : libavformat/rtpdec.c:812
c
static int rtp_parse_queued_packet(RTPDemuxContext *s, AVPacket *pkt)
{
int rv;
RTPPacket *next;
if (s->queue_len <= 0)
return -1;
// 1. 检查队首包是否是期望的下一个包
if (!has_next_packet(s)) {
int pkt_missed = s->queue->seq - s->seq - 1;
if (pkt_missed < 0)
pkt_missed += UINT16_MAX;
av_log(s->ic, AV_LOG_WARNING,
"RTP: missed %d packets\n", pkt_missed);
}
// 2. 解析队首包
rv = rtp_parse_packet_internal(s, pkt, s->queue->buf, s->queue->len);
next = s->queue->next;
// 3. 释放队首节点
av_freep(&s->queue->buf);
av_freep(&s->queue);
s->queue = next;
s->queue_len--;
return rv;
}重排序队列工作原理:
接收序列: 100, 102, 101, 103, 104
1. 收到seq=100: 直接输出 (队列为空)
2. 收到seq=102: 入队 (期望101)
队列: [102]
3. 收到seq=101: 直接输出 (期望的包)
然后输出队列中的102
队列: []
4. 收到seq=103: 直接输出
5. 收到seq=104: 直接输出五、Jitter计算
位置 : libavformat/rtpdec.c:295
c
static void rtcp_update_jitter(RTPStatistics *s, uint32_t sent_timestamp,
uint32_t arrival_timestamp)
{
// RFC 3550 Appendix A.8
uint32_t transit = arrival_timestamp - sent_timestamp;
uint32_t prev_transit = s->transit;
int32_t d = transit - prev_transit;
// 计算绝对值
d = FFABS(d);
s->transit = transit;
if (!prev_transit)
return;
// Jitter = Jitter + (|D| - Jitter) / 16
s->jitter += d - (int32_t) ((s->jitter + 8) >> 4);
}Jitter计算公式:
- Transit: 到达时间 - 发送时间
- D: 当前Transit - 上次Transit
- Jitter: 平滑的抖动估计 (指数加权移动平均)
六、完整示例流程
6.1 H.264视频帧接收示例
假设接收一个H.264 IDR帧，包含以下RTP包:
包1: SPS (Single NAL Unit)
RTP Header: V=2, PT=96, Seq=100, TS=12345, SSRC=0x12345678, M=0
Payload: [0x67 | SPS Data (7 bytes)]
解析结果:
AVPacket: [0x00 0x00 0x00 0x01 | 0x67 | SPS Data]包2: PPS (Single NAL Unit)
RTP Header: V=2, PT=96, Seq=101, TS=12345, SSRC=0x12345678, M=0
Payload: [0x68 | PPS Data (4 bytes)]
解析结果:
AVPacket: [0x00 0x00 0x00 0x01 | 0x68 | PPS Data]包3-5: IDR Slice (FU-A分片)
包3 (第一个分片):
RTP Header: V=2, PT=96, Seq=102, TS=12345, SSRC=0x12345678, M=0
Payload: [FU Indicator: 0x7C] [FU Header: 0x85] [Fragment 1 Data]
(Type=28, S=1, E=0, NAL Type=5)
解析结果:
AVPacket: [0x00 0x00 0x00 0x01 | 0x65 | Fragment 1 Data]
包4 (中间分片):
RTP Header: V=2, PT=96, Seq=103, TS=12345, SSRC=0x12345678, M=0
Payload: [FU Indicator: 0x7C] [FU Header: 0x05] [Fragment 2 Data]
(Type=28, S=0, E=0, NAL Type=5)
解析结果:
追加到现有AVPacket: [Fragment 2 Data]
包5 (最后分片):
RTP Header: V=2, PT=96, Seq=104, TS=12345, SSRC=0x12345678, M=1
Payload: [FU Indicator: 0x7C] [FU Header: 0x45] [Fragment 3 Data]
(Type=28, S=0, E=1, NAL Type=5)
解析结果:
追加到现有AVPacket: [Fragment 3 Data]
完整的IDR NAL单元组装完成6.2 AAC音频帧接收示例
单包多帧:
RTP Header: V=2, PT=97, Seq=200, TS=48000, SSRC=0x87654321, M=1
Payload:
AU Headers Length: 32 bits (0x0020)
AU Header 1: Size=200 (13 bits), Index=0 (3 bits)
AU Header 2: Size=180 (13 bits), Index=0 (3 bits)
AU Data 1: [200 bytes]
AU Data 2: [180 bytes]
解析结果:
第一次调用: 返回AU 1 (200 bytes)
第二次调用: 返回AU 2 (180 bytes)分片的AU:
包1 (第一个分片):
RTP Header: V=2, PT=97, Seq=201, TS=51024, SSRC=0x87654321, M=0
Payload:
AU Headers Length: 16 bits (0x0010)
AU Header: Size=2000 (13 bits), Index=0 (3 bits)
AU Data Fragment 1: [1400 bytes]
缓存Fragment 1
包2 (最后分片):
RTP Header: V=2, PT=97, Seq=202, TS=51024, SSRC=0x87654321, M=1
Payload:
AU Headers Length: 16 bits (0x0010)
AU Header: Size=2000 (13 bits), Index=0 (3 bits)
AU Data Fragment 2: [600 bytes]
组装完整AU: Fragment 1 + Fragment 2 = 2000 bytes七、关键数据结构
7.1 RTPDemuxContext
位置 : libavformat/rtpdec.h
c
struct RTPDemuxContext {
AVFormatContext *ic;
AVStream *st;
int payload_type;
uint32_t ssrc;
uint16_t seq;
uint32_t timestamp;
uint32_t base_timestamp;
int64_t unwrapped_timestamp;
int64_t range_start_offset;
// 重排序队列
RTPPacket *queue;
int queue_len;
int queue_size;
// 统计信息
RTPStatistics statistics;
// RTCP相关
AVRTCPSenderReport last_sr;
int64_t first_rtcp_ntp_time;
int64_t last_rtcp_reception_time;
int32_t rtcp_ts_offset;
int pending_sr;
// 编解码器特定handler
const RTPDynamicProtocolHandler *handler;
PayloadContext *dynamic_protocol_context;
// SRTP
struct SRTPContext srtp;
int srtp_enabled;
char hostname[256];
};7.2 RTPPacket (队列节点)
c
typedef struct RTPPacket {
uint16_t seq;
uint8_t *buf;
int len;
int64_t recvtime;
struct RTPPacket *next;
} RTPPacket;7.3 RTPStatistics
c
typedef struct RTPStatistics {
uint16_t max_seq; // 最大序列号
uint32_t cycles; // 序列号回绕次数
uint32_t base_seq; // 基础序列号
uint32_t bad_seq; // 上次"坏"序列号
uint32_t probation; // Probation期剩余包数
uint32_t received; // 接收包数
uint32_t expected_prior; // 上次期望包数
uint32_t received_prior; // 上次接收包数
uint32_t transit; // 上次Transit值
uint32_t jitter; // 抖动估计
} RTPStatistics;八、错误处理和容错机制
8.1 丢包检测
c
// 在rtp_parse_queued_packet中
if (!has_next_packet(s)) {
int pkt_missed = s->queue->seq - s->seq - 1;
if (pkt_missed < 0)
pkt_missed += UINT16_MAX;
av_log(s->ic, AV_LOG_WARNING,
"RTP: missed %d packets\n", pkt_missed);
}8.2 乱序处理
- 小乱序 (diff <= 1): 直接输出
- 大乱序 (diff > 1): 入队等待
- 队列满: 强制输出最早的包
8.3 超时处理
c
// 在ff_rtsp_fetch_packet中
if (first_queue_time) {
wait_end = first_queue_time + s->max_delay;
}
if (len == AVERROR(EAGAIN) && first_queue_st) {
av_log(s, AV_LOG_WARNING,
"max delay reached. need to consume packet\n");
// 强制输出队列中的包
}8.4 传输层切换
c
// UDP超时自动切换到TCP
if (ret == AVERROR(ETIMEDOUT) && !rt->packets) {
if (rt->lower_transport == RTSP_LOWER_TRANSPORT_UDP &&
rt->lower_transport_mask & (1 << RTSP_LOWER_TRANSPORT_TCP)) {
av_log(s, AV_LOG_WARNING, "UDP timeout, retrying with TCP\n");
if (resetup_tcp(s) == 0) {
// 重新SETUP和PLAY
}
}
}九、性能优化
9.1 零拷贝机制
- 入队: 直接保存buf指针，避免复制
- 出队: 转移buf所有权到AVPacket
9.2 缓存机制
- cur_transport_priv: 缓存当前正在解析的流
- 避免重复查找和解析
9.3 批量处理
- 多AU包: 一次RTP包解析出多个AVPacket
- STAP-A: 一次解析多个NAL单元
十、总结
RTSP拉流RTP包解析的完整流程:
- 网络接收: UDP/TCP Interleaved
- 入口函数 :
rtsp_read_packet()→ff_rtsp_fetch_packet() - RTP解析 :
ff_rtp_parse_packet()→rtp_parse_packet_internal() - 头部解析 :
- RTP固定头 (12字节)
- 序列号验证
- Padding/CSRC/Extension处理
- 编解码器解包 :
- H.264: Single NAL/STAP-A/FU-A
- AAC: AU Header解析 + 多帧/分片处理
- 时间戳处理 :
finalize_packet()- 基于RTCP SR (多流同步)
- 基于RTP时间戳 (单流)
- 容错机制 :
- 重排序队列
- 丢包检测
- 超时处理
- 传输层切换
整个过程确保了:
- 可靠性: 序列号验证、丢包检测
- 实时性: Jitter buffer、超时机制
- 同步性: RTCP SR时间戳同步
- 兼容性: 多种编解码器和传输模式