音视频入门基础：MPEG2-TS专题（19）——FFmpeg源码中，解析TS流中的PES流的实现

cuijiecheng20182024-12-19 19:20

一、引言

FFmpeg源码在解析完PMT表后，会得到该节目包含的视频和音频信息，从而找到音视频流。TS流的音视频流包含在PES流中。FFmpeg源码通过调用函数指针tss->u.pes_filter.pes_cb指向的回调函数解析PES流的PES packet：

cpp 复制代码 
/* handle one TS packet */
static int handle_packet(MpegTSContext *ts, const uint8_t *packet, int64_t pos)
{
//...
    if (tss->type == MPEGTS_SECTION) {
    //...
    }
    else {
        int ret;
        // Note: The position here points actually behind the current packet.
        if (tss->type == MPEGTS_PES) {
            if ((ret = tss->u.pes_filter.pes_cb(tss, p, p_end - p, is_start,
                                                pos - ts->raw_packet_size)) < 0)
                return ret;
        }
    }

    return 0;
//...
}

而函数指针tss->u.pes_filter.pes_cb指向的回调函数是mpegts_push_data函数。

二、mpegts_push_data函数的定义

mpegts_push_data函数定义在FFmpeg源码（本文演示用的FFmpeg源码版本为7.0.1）的源文件libavformat/mpegts.c中：

cpp 复制代码 
/* return non zero if a packet could be constructed */
static int mpegts_push_data(MpegTSFilter *filter,
                            const uint8_t *buf, int buf_size, int is_start,
                            int64_t pos)
{
    PESContext *pes   = filter->u.pes_filter.opaque;
    MpegTSContext *ts = pes->ts;
    const uint8_t *p;
    int ret, len;

    if (!ts->pkt)
        return 0;

    if (is_start) {
        if (pes->state == MPEGTS_PAYLOAD && pes->data_index > 0) {
            ret = new_pes_packet(pes, ts->pkt);
            if (ret < 0)
                return ret;
            ts->stop_parse = 1;
        } else {
            reset_pes_packet_state(pes);
        }
        pes->state         = MPEGTS_HEADER;
        pes->ts_packet_pos = pos;
    }
    p = buf;
    while (buf_size > 0) {
        switch (pes->state) {
        case MPEGTS_HEADER:
            len = PES_START_SIZE - pes->data_index;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;
            if (pes->data_index == PES_START_SIZE) {
                /* we got all the PES or section header. We can now
                 * decide */
                if (pes->header[0] == 0x00 && pes->header[1] == 0x00 &&
                    pes->header[2] == 0x01) {
                    /* it must be an MPEG-2 PES stream */
                    pes->stream_id = pes->header[3];
                    av_log(pes->stream, AV_LOG_TRACE, "pid=%x stream_id=%#x\n", pes->pid, pes->stream_id);

                    if ((pes->st && pes->st->discard == AVDISCARD_ALL &&
                         (!pes->sub_st ||
                          pes->sub_st->discard == AVDISCARD_ALL)) ||
                        pes->stream_id == STREAM_ID_PADDING_STREAM)
                        goto skip;

                    /* stream not present in PMT */
                    if (!pes->st) {
                        if (ts->skip_changes)
                            goto skip;
                        if (ts->merge_pmt_versions)
                            goto skip; /* wait for PMT to merge new stream */

                        pes->st = avformat_new_stream(ts->stream, NULL);
                        if (!pes->st)
                            return AVERROR(ENOMEM);
                        pes->st->id = pes->pid;
                        mpegts_set_stream_info(pes->st, pes, 0, 0);
                    }

                    pes->PES_packet_length = AV_RB16(pes->header + 4);
                    /* NOTE: zero length means the PES size is unbounded */

                    if (pes->stream_id != STREAM_ID_PROGRAM_STREAM_MAP &&
                        pes->stream_id != STREAM_ID_PRIVATE_STREAM_2 &&
                        pes->stream_id != STREAM_ID_ECM_STREAM &&
                        pes->stream_id != STREAM_ID_EMM_STREAM &&
                        pes->stream_id != STREAM_ID_PROGRAM_STREAM_DIRECTORY &&
                        pes->stream_id != STREAM_ID_DSMCC_STREAM &&
                        pes->stream_id != STREAM_ID_TYPE_E_STREAM) {
                        FFStream *const pes_sti = ffstream(pes->st);
                        pes->state = MPEGTS_PESHEADER;
                        if (pes->st->codecpar->codec_id == AV_CODEC_ID_NONE && !pes_sti->request_probe) {
                            av_log(pes->stream, AV_LOG_TRACE,
                                    "pid=%x stream_type=%x probing\n",
                                    pes->pid,
                                    pes->stream_type);
                            pes_sti->request_probe = 1;
                        }
                    } else {
                        pes->pes_header_size = 6;
                        pes->state      = MPEGTS_PAYLOAD;
                        pes->data_index = 0;
                    }
                } else {
                    /* otherwise, it should be a table */
                    /* skip packet */
skip:
                    pes->state = MPEGTS_SKIP;
                    continue;
                }
            }
            break;
        /**********************************************/
        /* PES packing parsing */
        case MPEGTS_PESHEADER:
            len = PES_HEADER_SIZE - pes->data_index;
            if (len < 0)
                return AVERROR_INVALIDDATA;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;
            if (pes->data_index == PES_HEADER_SIZE) {
                pes->pes_header_size = pes->header[8] + 9;
                pes->state           = MPEGTS_PESHEADER_FILL;
            }
            break;
        case MPEGTS_PESHEADER_FILL:
            len = pes->pes_header_size - pes->data_index;
            if (len < 0)
                return AVERROR_INVALIDDATA;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;
            if (pes->data_index == pes->pes_header_size) {
                const uint8_t *r;
                unsigned int flags, pes_ext, skip;

                flags = pes->header[7];
                r = pes->header + 9;
                pes->pts = AV_NOPTS_VALUE;
                pes->dts = AV_NOPTS_VALUE;
                if ((flags & 0xc0) == 0x80) {
                    pes->dts = pes->pts = ff_parse_pes_pts(r);
                    r += 5;
                } else if ((flags & 0xc0) == 0xc0) {
                    pes->pts = ff_parse_pes_pts(r);
                    r += 5;
                    pes->dts = ff_parse_pes_pts(r);
                    r += 5;
                }
                pes->extended_stream_id = -1;
                if (flags & 0x01) { /* PES extension */
                    pes_ext = *r++;
                    /* Skip PES private data, program packet sequence counter and P-STD buffer */
                    skip  = (pes_ext >> 4) & 0xb;
                    skip += skip & 0x9;
                    r    += skip;
                    if ((pes_ext & 0x41) == 0x01 &&
                        (r + 2) <= (pes->header + pes->pes_header_size)) {
                        /* PES extension 2 */
                        if ((r[0] & 0x7f) > 0 && (r[1] & 0x80) == 0)
                            pes->extended_stream_id = r[1];
                    }
                }

                /* we got the full header. We parse it and get the payload */
                pes->state = MPEGTS_PAYLOAD;
                pes->data_index = 0;
                if (pes->stream_type == 0x12 && buf_size > 0) {
                    int sl_header_bytes = read_sl_header(pes, &pes->sl, p,
                                                         buf_size);
                    pes->pes_header_size += sl_header_bytes;
                    p += sl_header_bytes;
                    buf_size -= sl_header_bytes;
                }
                if (pes->stream_type == STREAM_TYPE_METADATA &&
                    pes->stream_id == STREAM_ID_METADATA_STREAM &&
                    pes->st->codecpar->codec_id == AV_CODEC_ID_SMPTE_KLV &&
                    buf_size >= 5) {
                    /* skip metadata access unit header - see MISB ST 1402 */
                    pes->pes_header_size += 5;
                    p += 5;
                    buf_size -= 5;
                }
                if (   pes->ts->fix_teletext_pts
                    && (   pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_TELETEXT
                        || pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_SUBTITLE)
                    ) {
                    AVProgram *p = NULL;
                    int pcr_found = 0;
                    while ((p = av_find_program_from_stream(pes->stream, p, pes->st->index))) {
                        if (p->pcr_pid != -1 && p->discard != AVDISCARD_ALL) {
                            MpegTSFilter *f = pes->ts->pids[p->pcr_pid];
                            if (f) {
                                AVStream *st = NULL;
                                if (f->type == MPEGTS_PES) {
                                    PESContext *pcrpes = f->u.pes_filter.opaque;
                                    if (pcrpes)
                                        st = pcrpes->st;
                                } else if (f->type == MPEGTS_PCR) {
                                    int i;
                                    for (i = 0; i < p->nb_stream_indexes; i++) {
                                        AVStream *pst = pes->stream->streams[p->stream_index[i]];
                                        if (pst->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
                                            st = pst;
                                    }
                                }
                                if (f->last_pcr != -1 && !f->discard) {
                                    // teletext packets do not always have correct timestamps,
                                    // the standard says they should be handled after 40.6 ms at most,
                                    // and the pcr error to this packet should be no more than 100 ms.
                                    // TODO: we should interpolate the PCR, not just use the last one
                                    int64_t pcr = f->last_pcr / 300;
                                    pcr_found = 1;
                                    if (st) {
                                        const FFStream *const sti = ffstream(st);
                                        FFStream *const pes_sti   = ffstream(pes->st);

                                        pes_sti->pts_wrap_reference = sti->pts_wrap_reference;
                                        pes_sti->pts_wrap_behavior  = sti->pts_wrap_behavior;
                                    }
                                    if (pes->dts == AV_NOPTS_VALUE || pes->dts < pcr) {
                                        pes->pts = pes->dts = pcr;
                                    } else if (pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_TELETEXT &&
                                               pes->dts > pcr + 3654 + 9000) {
                                        pes->pts = pes->dts = pcr + 3654 + 9000;
                                    } else if (pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_SUBTITLE &&
                                               pes->dts > pcr + 10*90000) { //10sec
                                        pes->pts = pes->dts = pcr + 3654 + 9000;
                                    }
                                    break;
                                }
                            }
                        }
                    }

                    if (pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_TELETEXT &&
                        !pcr_found) {
                        av_log(pes->stream, AV_LOG_VERBOSE,
                               "Forcing DTS/PTS to be unset for a "
                               "non-trustworthy PES packet for PID %d as "
                               "PCR hasn't been received yet.\n",
                               pes->pid);
                        pes->dts = pes->pts = AV_NOPTS_VALUE;
                    }
                }
            }
            break;
        case MPEGTS_PAYLOAD:
            do {
                int max_packet_size = ts->max_packet_size;
                if (pes->PES_packet_length && pes->PES_packet_length + PES_START_SIZE > pes->pes_header_size)
                    max_packet_size = pes->PES_packet_length + PES_START_SIZE - pes->pes_header_size;

                if (pes->data_index > 0 &&
                    pes->data_index + buf_size > max_packet_size) {
                    ret = new_pes_packet(pes, ts->pkt);
                    if (ret < 0)
                        return ret;
                    pes->PES_packet_length = 0;
                    max_packet_size = ts->max_packet_size;
                    ts->stop_parse = 1;
                } else if (pes->data_index == 0 &&
                           buf_size > max_packet_size) {
                    // pes packet size is < ts size packet and pes data is padded with 0xff
                    // not sure if this is legal in ts but see issue #2392
                    buf_size = max_packet_size;
                }

                if (!pes->buffer) {
                    pes->buffer = buffer_pool_get(ts, max_packet_size);
                    if (!pes->buffer)
                        return AVERROR(ENOMEM);
                }

                memcpy(pes->buffer->data + pes->data_index, p, buf_size);
                pes->data_index += buf_size;
                /* emit complete packets with known packet size
                 * decreases demuxer delay for infrequent packets like subtitles from
                 * a couple of seconds to milliseconds for properly muxed files. */
                if (!ts->stop_parse && pes->PES_packet_length &&
                    pes->pes_header_size + pes->data_index == pes->PES_packet_length + PES_START_SIZE) {
                    ts->stop_parse = 1;
                    ret = new_pes_packet(pes, ts->pkt);
                    pes->state = MPEGTS_SKIP;
                    if (ret < 0)
                        return ret;
                }
            } while (0);
            buf_size = 0;
            break;
        case MPEGTS_SKIP:
            buf_size = 0;
            break;
        }
    }

    return 0;
}

该函数的作用就是：解析TS流中的PES流的PES packet（PES包）。即解析TS流中的一个PES packet。

形参filter：输出型参数，指向一个MpegTSFilter类型变量。

形参buf：输入型参数。存放一个transport packet（TS包）去掉TS Header后的有效数据，即ES packet的数据。

形参buf_size：输入型参数。形参buf指向的缓冲区的大小。

形参is_start：输入型参数。如果值为true，表示该transport packet的TS Header的payload_unit_start_indicator属性的值为1，说明携带的是PES的第一个包。

形参pos：输入型参数，文件位置指针当前位置相对于TS文件的文件首的偏移字节数 减去 188字节。

返回值：返回0表示解析成功，返回一个负数表示出错。

三、mpegts_push_data函数的内部实现分析

mpegts_push_data函数内部使用了状态机的设计模式，会根据枚举变量pes->state的值执行不同的处理逻辑，pes->state取值如下：

cpp 复制代码 
/* TS stream handling */

enum MpegTSState {
    MPEGTS_HEADER = 0,
    MPEGTS_PESHEADER,
    MPEGTS_PESHEADER_FILL,
    MPEGTS_PAYLOAD,
    MPEGTS_SKIP,
};

mpegts_push_data函数中首先判断形参is_start的值，如果为true，说明携带的是PES的第一个包。让pes->state赋值为MPEGTS_HEADER：

cpp 复制代码 
    if (is_start) {
        if (pes->state == MPEGTS_PAYLOAD && pes->data_index > 0) {
            ret = new_pes_packet(pes, ts->pkt);
            if (ret < 0)
                return ret;
            ts->stop_parse = 1;
        } else {
            reset_pes_packet_state(pes);
        }
        pes->state         = MPEGTS_HEADER;
        pes->ts_packet_pos = pos;
    }

下面分情况讨论。

（一）pes->state的值为MPEGTS_HEADER

pes->state的值为MPEGTS_HEADER时，mpegts_push_data函数会执行下面代码块解析该PES packet的PES packet header中的固定长度部分：

cpp 复制代码 
        switch (pes->state) {
        case MPEGTS_HEADER:
            len = PES_START_SIZE - pes->data_index;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;
            if (pes->data_index == PES_START_SIZE) {
                /* we got all the PES or section header. We can now
                 * decide */
                if (pes->header[0] == 0x00 && pes->header[1] == 0x00 &&
                    pes->header[2] == 0x01) {
                    /* it must be an MPEG-2 PES stream */
                    pes->stream_id = pes->header[3];
                    av_log(pes->stream, AV_LOG_TRACE, "pid=%x stream_id=%#x\n", pes->pid, pes->stream_id);

                    if ((pes->st && pes->st->discard == AVDISCARD_ALL &&
                         (!pes->sub_st ||
                          pes->sub_st->discard == AVDISCARD_ALL)) ||
                        pes->stream_id == STREAM_ID_PADDING_STREAM)
                        goto skip;

                    /* stream not present in PMT */
                    if (!pes->st) {
                        if (ts->skip_changes)
                            goto skip;
                        if (ts->merge_pmt_versions)
                            goto skip; /* wait for PMT to merge new stream */

                        pes->st = avformat_new_stream(ts->stream, NULL);
                        if (!pes->st)
                            return AVERROR(ENOMEM);
                        pes->st->id = pes->pid;
                        mpegts_set_stream_info(pes->st, pes, 0, 0);
                    }

                    pes->PES_packet_length = AV_RB16(pes->header + 4);
                    /* NOTE: zero length means the PES size is unbounded */

                    if (pes->stream_id != STREAM_ID_PROGRAM_STREAM_MAP &&
                        pes->stream_id != STREAM_ID_PRIVATE_STREAM_2 &&
                        pes->stream_id != STREAM_ID_ECM_STREAM &&
                        pes->stream_id != STREAM_ID_EMM_STREAM &&
                        pes->stream_id != STREAM_ID_PROGRAM_STREAM_DIRECTORY &&
                        pes->stream_id != STREAM_ID_DSMCC_STREAM &&
                        pes->stream_id != STREAM_ID_TYPE_E_STREAM) {
                        FFStream *const pes_sti = ffstream(pes->st);
                        pes->state = MPEGTS_PESHEADER;
                        if (pes->st->codecpar->codec_id == AV_CODEC_ID_NONE && !pes_sti->request_probe) {
                            av_log(pes->stream, AV_LOG_TRACE,
                                    "pid=%x stream_type=%x probing\n",
                                    pes->pid,
                                    pes->stream_type);
                            pes_sti->request_probe = 1;
                        }
                    } else {
                        pes->pes_header_size = 6;
                        pes->state      = MPEGTS_PAYLOAD;
                        pes->data_index = 0;
                    }
                } else {
                    /* otherwise, it should be a table */
                    /* skip packet */
skip:
                    pes->state = MPEGTS_SKIP;
                    continue;
                }
            }

宏PES_START_SIZE定义在libavformat/mpegts.c中，值为6，表示PES packet header中的固定长度部分总共占6个字节：

cpp 复制代码 
#define PES_START_SIZE  6

上述代码块中，首先将该PES packet的PES packet header中的固定长度部分拷贝到数组pes->header中：

cpp 复制代码 
            len = PES_START_SIZE - pes->data_index;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;

判断PES packet header中的packet_start_code_prefix属性的值是否为0x000001，如果是，说明这是PES packet，读取stream_id属性的值，赋值给变量pes->stream_id：

cpp 复制代码 
                /* we got all the PES or section header. We can now
                 * decide */
                if (pes->header[0] == 0x00 && pes->header[1] == 0x00 &&
                    pes->header[2] == 0x01) {
                    /* it must be an MPEG-2 PES stream */
                    pes->stream_id = pes->header[3];
                    av_log(pes->stream, AV_LOG_TRACE, "pid=%x stream_id=%#x\n", pes->pid, pes->stream_id);
                //...
}

如果在之前解析PMT表的时候没有找到该PES流对应的音视频流，跳过本次循环：

cpp 复制代码 
                    /* stream not present in PMT */
                    if (!pes->st) {
                        if (ts->skip_changes)
                            goto skip;
                        if (ts->merge_pmt_versions)
                            goto skip; /* wait for PMT to merge new stream */

                        pes->st = avformat_new_stream(ts->stream, NULL);
                        if (!pes->st)
                            return AVERROR(ENOMEM);
                        pes->st->id = pes->pid;
                        mpegts_set_stream_info(pes->st, pes, 0, 0);
                    }

读取PES_packet_length属性的值，赋值给变量pes->PES_packet_length：

cpp 复制代码 
                    pes->PES_packet_length = AV_RB16(pes->header + 4);
                    /* NOTE: zero length means the PES size is unbounded */

通过PES packet header中的stream_id的值判断PES packet header中是否存在Optional PES header，如果存在，让pes->state赋值为MPEGTS_PESHEADER：

cpp 复制代码 
                    if (pes->stream_id != STREAM_ID_PROGRAM_STREAM_MAP &&
                        pes->stream_id != STREAM_ID_PRIVATE_STREAM_2 &&
                        pes->stream_id != STREAM_ID_ECM_STREAM &&
                        pes->stream_id != STREAM_ID_EMM_STREAM &&
                        pes->stream_id != STREAM_ID_PROGRAM_STREAM_DIRECTORY &&
                        pes->stream_id != STREAM_ID_DSMCC_STREAM &&
                        pes->stream_id != STREAM_ID_TYPE_E_STREAM) {
                        FFStream *const pes_sti = ffstream(pes->st);
                        pes->state = MPEGTS_PESHEADER;
                        if (pes->st->codecpar->codec_id == AV_CODEC_ID_NONE && !pes_sti->request_probe) {
                            av_log(pes->stream, AV_LOG_TRACE,
                                    "pid=%x stream_type=%x probing\n",
                                    pes->pid,
                                    pes->stream_type);
                            pes_sti->request_probe = 1;
                        }
                    } else {
                        pes->pes_header_size = 6;
                        pes->state      = MPEGTS_PAYLOAD;
                        pes->data_index = 0;
                    }

（二）pes->state的值为MPEGTS_PESHEADER

pes->state的值为MPEGTS_PESHEADER时，mpegts_push_data函数会执行下面代码块：

cpp 复制代码 
        /**********************************************/
        /* PES packing parsing */
        case MPEGTS_PESHEADER:
            len = PES_HEADER_SIZE - pes->data_index;
            if (len < 0)
                return AVERROR_INVALIDDATA;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;
            if (pes->data_index == PES_HEADER_SIZE) {
                pes->pes_header_size = pes->header[8] + 9;
                pes->state           = MPEGTS_PESHEADER_FILL;
            }
            break;

宏PES_HEADER_SIZE定义在libavformat/mpegts.c中，值为9，表示PES packet header中的固定长度部分 + Optional PES header中的PES_header_data_length属性之前的部分（包含PES_header_data_length属性）总共占9个字节：

cpp 复制代码 
#define PES_HEADER_SIZE 9

上述代码块中，首先将该PES packet的Optional PES header中的PES_header_data_length属性之前的部分（包含PES_header_data_length属性）拷贝到pes->header + pes->data_index指向的缓冲区中：

cpp 复制代码 
            len = PES_HEADER_SIZE - pes->data_index;
            if (len < 0)
                return AVERROR_INVALIDDATA;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;

读取Optional PES header中的PES_header_data_length属性，赋值给变量pes->pes_header_size。让变量pes->state赋值为MPEGTS_PESHEADER_FILL：

cpp 复制代码 
            if (pes->data_index == PES_HEADER_SIZE) {
                pes->pes_header_size = pes->header[8] + 9;
                pes->state           = MPEGTS_PESHEADER_FILL;
            }

（三）pes->state的值为MPEGTS_PESHEADER_FILL

pes->state的值为MPEGTS_PESHEADER_FILL时，mpegts_push_data函数会执行下面代码块：

cpp 复制代码 
            len = pes->pes_header_size - pes->data_index;
            if (len < 0)
                return AVERROR_INVALIDDATA;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;
            if (pes->data_index == pes->pes_header_size) {
                const uint8_t *r;
                unsigned int flags, pes_ext, skip;

                flags = pes->header[7];
                r = pes->header + 9;
                pes->pts = AV_NOPTS_VALUE;
                pes->dts = AV_NOPTS_VALUE;
                if ((flags & 0xc0) == 0x80) {
                    pes->dts = pes->pts = ff_parse_pes_pts(r);
                    r += 5;
                } else if ((flags & 0xc0) == 0xc0) {
                    pes->pts = ff_parse_pes_pts(r);
                    r += 5;
                    pes->dts = ff_parse_pes_pts(r);
                    r += 5;
                }
                pes->extended_stream_id = -1;
                if (flags & 0x01) { /* PES extension */
                    pes_ext = *r++;
                    /* Skip PES private data, program packet sequence counter and P-STD buffer */
                    skip  = (pes_ext >> 4) & 0xb;
                    skip += skip & 0x9;
                    r    += skip;
                    if ((pes_ext & 0x41) == 0x01 &&
                        (r + 2) <= (pes->header + pes->pes_header_size)) {
                        /* PES extension 2 */
                        if ((r[0] & 0x7f) > 0 && (r[1] & 0x80) == 0)
                            pes->extended_stream_id = r[1];
                    }
                }

                /* we got the full header. We parse it and get the payload */
                pes->state = MPEGTS_PAYLOAD;
                pes->data_index = 0;
                if (pes->stream_type == 0x12 && buf_size > 0) {
                    int sl_header_bytes = read_sl_header(pes, &pes->sl, p,
                                                         buf_size);
                    pes->pes_header_size += sl_header_bytes;
                    p += sl_header_bytes;
                    buf_size -= sl_header_bytes;
                }
                if (pes->stream_type == STREAM_TYPE_METADATA &&
                    pes->stream_id == STREAM_ID_METADATA_STREAM &&
                    pes->st->codecpar->codec_id == AV_CODEC_ID_SMPTE_KLV &&
                    buf_size >= 5) {
                    /* skip metadata access unit header - see MISB ST 1402 */
                    pes->pes_header_size += 5;
                    p += 5;
                    buf_size -= 5;
                }
                if (   pes->ts->fix_teletext_pts
                    && (   pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_TELETEXT
                        || pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_SUBTITLE)
                    ) {
                    AVProgram *p = NULL;
                    int pcr_found = 0;
                    while ((p = av_find_program_from_stream(pes->stream, p, pes->st->index))) {
                        if (p->pcr_pid != -1 && p->discard != AVDISCARD_ALL) {
                            MpegTSFilter *f = pes->ts->pids[p->pcr_pid];
                            if (f) {
                                AVStream *st = NULL;
                                if (f->type == MPEGTS_PES) {
                                    PESContext *pcrpes = f->u.pes_filter.opaque;
                                    if (pcrpes)
                                        st = pcrpes->st;
                                } else if (f->type == MPEGTS_PCR) {
                                    int i;
                                    for (i = 0; i < p->nb_stream_indexes; i++) {
                                        AVStream *pst = pes->stream->streams[p->stream_index[i]];
                                        if (pst->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
                                            st = pst;
                                    }
                                }
                                if (f->last_pcr != -1 && !f->discard) {
                                    // teletext packets do not always have correct timestamps,
                                    // the standard says they should be handled after 40.6 ms at most,
                                    // and the pcr error to this packet should be no more than 100 ms.
                                    // TODO: we should interpolate the PCR, not just use the last one
                                    int64_t pcr = f->last_pcr / 300;
                                    pcr_found = 1;
                                    if (st) {
                                        const FFStream *const sti = ffstream(st);
                                        FFStream *const pes_sti   = ffstream(pes->st);

                                        pes_sti->pts_wrap_reference = sti->pts_wrap_reference;
                                        pes_sti->pts_wrap_behavior  = sti->pts_wrap_behavior;
                                    }
                                    if (pes->dts == AV_NOPTS_VALUE || pes->dts < pcr) {
                                        pes->pts = pes->dts = pcr;
                                    } else if (pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_TELETEXT &&
                                               pes->dts > pcr + 3654 + 9000) {
                                        pes->pts = pes->dts = pcr + 3654 + 9000;
                                    } else if (pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_SUBTITLE &&
                                               pes->dts > pcr + 10*90000) { //10sec
                                        pes->pts = pes->dts = pcr + 3654 + 9000;
                                    }
                                    break;
                                }
                            }
                        }
                    }

                    if (pes->st->codecpar->codec_id == AV_CODEC_ID_DVB_TELETEXT &&
                        !pcr_found) {
                        av_log(pes->stream, AV_LOG_VERBOSE,
                               "Forcing DTS/PTS to be unset for a "
                               "non-trustworthy PES packet for PID %d as "
                               "PCR hasn't been received yet.\n",
                               pes->pid);
                        pes->dts = pes->pts = AV_NOPTS_VALUE;
                    }
                }
            }
            break;

上述代码块中，首先将该PES packet的Optional PES header中的可选字段占用的总字节数，以及包含在此PES packet header的任何填充字节拷贝到pes->header + pes->data_index指向的缓冲区中：

cpp 复制代码 
            len = pes->pes_header_size - pes->data_index;
            if (len < 0)
                return AVERROR_INVALIDDATA;
            if (len > buf_size)
                len = buf_size;
            memcpy(pes->header + pes->data_index, p, len);
            pes->data_index += len;
            p += len;
            buf_size -= len;

读取Optional PES header中的PTS_DTS_flags、ESCR_flag、ES_rate_flag、DSM_trick_mode_flag、additional_copy_info_flag、PES_CRC_flag、PES_extension_flag这7个属性（这7个属性加起来总共1个字节），赋值给变量flags：

cpp 复制代码 
                const uint8_t *r;
                unsigned int flags, pes_ext, skip;

                flags = pes->header[7];
                r = pes->header + 9;
                pes->pts = AV_NOPTS_VALUE;
                pes->dts = AV_NOPTS_VALUE;

如果Optional PES header中的PTS_DTS_flags属性的值为'10'，表示PES packet header中会存在PTS。读取PTS的值赋值给变量pes->pts，让pes->dts也等于PTS：

cpp 复制代码 
                if ((flags & 0xc0) == 0x80) {
                    pes->dts = pes->pts = ff_parse_pes_pts(r);
                    r += 5;
                }

如果PTS_DTS_flags属性的值为'11'，PES packet header中会同时存在PTS和DTS，读取PTS的值赋值给变量pes->pts，读取DTS的值赋值给变量pes->dts：

cpp 复制代码 
                if ((flags & 0xc0) == 0x80) {
                //...
                } else if ((flags & 0xc0) == 0xc0) {
                    pes->pts = ff_parse_pes_pts(r);
                    r += 5;
                    pes->dts = ff_parse_pes_pts(r);
                    r += 5;
                }

如果PES_extension_flag属性的值为1，表示PES packet header有PES_extension域，解析PES_extension域：

cpp 复制代码 
                if (flags & 0x01) { /* PES extension */
                    pes_ext = *r++;
                    /* Skip PES private data, program packet sequence counter and P-STD buffer */
                    skip  = (pes_ext >> 4) & 0xb;
                    skip += skip & 0x9;
                    r    += skip;
                    if ((pes_ext & 0x41) == 0x01 &&
                        (r + 2) <= (pes->header + pes->pes_header_size)) {
                        /* PES extension 2 */
                        if ((r[0] & 0x7f) > 0 && (r[1] & 0x80) == 0)
                            pes->extended_stream_id = r[1];
                    }
                }

至此，mpegts_push_data函数已解析完整个PES packet header，让变量pes->state赋值为MPEGTS_PAYLOAD：

cpp 复制代码 
                /* we got the full header. We parse it and get the payload */
                pes->state = MPEGTS_PAYLOAD;
                pes->data_index = 0;

（四）pes->state的值为MPEGTS_PAYLOAD

pes->state的值为MPEGTS_PAYLOAD时，mpegts_push_data函数会执行下面代码块：

cpp 复制代码 
case MPEGTS_PAYLOAD:
            do {
                int max_packet_size = ts->max_packet_size;
                if (pes->PES_packet_length && pes->PES_packet_length + PES_START_SIZE > pes->pes_header_size)
                    max_packet_size = pes->PES_packet_length + PES_START_SIZE - pes->pes_header_size;

                if (pes->data_index > 0 &&
                    pes->data_index + buf_size > max_packet_size) {
                    ret = new_pes_packet(pes, ts->pkt);
                    if (ret < 0)
                        return ret;
                    pes->PES_packet_length = 0;
                    max_packet_size = ts->max_packet_size;
                    ts->stop_parse = 1;
                } else if (pes->data_index == 0 &&
                           buf_size > max_packet_size) {
                    // pes packet size is < ts size packet and pes data is padded with 0xff
                    // not sure if this is legal in ts but see issue #2392
                    buf_size = max_packet_size;
                }

                if (!pes->buffer) {
                    pes->buffer = buffer_pool_get(ts, max_packet_size);
                    if (!pes->buffer)
                        return AVERROR(ENOMEM);
                }

                memcpy(pes->buffer->data + pes->data_index, p, buf_size);
                pes->data_index += buf_size;
                /* emit complete packets with known packet size
                 * decreases demuxer delay for infrequent packets like subtitles from
                 * a couple of seconds to milliseconds for properly muxed files. */
                if (!ts->stop_parse && pes->PES_packet_length &&
                    pes->pes_header_size + pes->data_index == pes->PES_packet_length + PES_START_SIZE) {
                    ts->stop_parse = 1;
                    ret = new_pes_packet(pes, ts->pkt);
                    pes->state = MPEGTS_SKIP;
                    if (ret < 0)
                        return ret;
                }
            } while (0);
            buf_size = 0;
            break;

上述代码块的主要作用就是将该PES packet的PES packet data bytes（PES包的负载）拷贝到pes->buffer->data + pes->data_index指向的缓冲区中：

cpp 复制代码 
                memcpy(pes->buffer->data + pes->data_index, p, buf_size);
                pes->data_index += buf_size;

ts->pkt指向一个AVPacket类型的变量，让ts->pkt得到该PES packet的数据：

cpp 复制代码 
                /* emit complete packets with known packet size
                 * decreases demuxer delay for infrequent packets like subtitles from
                 * a couple of seconds to milliseconds for properly muxed files. */
                if (!ts->stop_parse && pes->PES_packet_length &&
                    pes->pes_header_size + pes->data_index == pes->PES_packet_length + PES_START_SIZE) {
                    ts->stop_parse = 1;
                    ret = new_pes_packet(pes, ts->pkt);
                    pes->state = MPEGTS_SKIP;
                    if (ret < 0)
                        return ret;
                }
相关推荐
ZVAyIVqt0UFji2 小时前
如何使用whisper+ollama+ffmpeg为视频添加中文字幕
ffmpeg·whisper·音视频
YRr YRr2 小时前
解决 OpenCV 与 FFmpeg 版本不兼容导致的编译错误
人工智能·opencv·ffmpeg
ai产品老杨5 小时前
提前对风险进行预警并实施管控，运用AI技术将管理推向新时代的智慧地产开源了。
vue.js·人工智能·安全·开源·音视频
静止了 所有的花开11 小时前
Onvif服务端开发
音视频·onvif
野蛮的大西瓜13 小时前
BigBlueButton目前支持哪些操作系统和浏览器
人工智能·机器人·自动化·音视频·信息与通信
特立独行的猫a13 小时前
HarmonyOS ArkTS中视频播放Video组件实现竖屏到横屏切换
华为·音视频·harmonyos
Lunar*15 小时前
使用 NVIDIA DALI 计算视频的光流
python·音视频
orangapple17 小时前
WPF 用Vlc.DotNet.Wpf实现视频播放、停止、暂停功能
wpf·音视频
haibo214421 小时前
Ruyi-Mini-7B：开源的图像生成视频模型
音视频
热门推荐
01YOLOv8训练好模型后，追加轮数继续训练、或者提前终止训练，缩减训练轮数02玄机平台应急响应—webshell查杀03(欧拉)openEuler系统添加网卡文件配置流程、(欧拉)openEuler系统手动配置ipv6地址流程、(欧拉)openEuler系统网络管理说明04【OCPP】ocpp1.6协议第3.13章节SmartCharging介绍及翻译05HCIA-datacom数通题库和录播视频资料06Docker 夺命连环 15 问07Rime 如何通过 iCloud 实现词库多端同步，Windows、iOS、macOS08WPS-0DAY-20230809的分析和利用复现09PyTorch机器学习实现液态神经网络10【机器学习】探索机器学习在旅游业的革新之旅