ffmpeg7.0 flv支持hdr

ffmpeg7.0 flv支持hdr

自从ffmpeg6.0应用enhance rtmp支持h265/av1的flv格式后，7.0迎来了flv的hdr能力。本文介绍ffmpeg7.0如何支持hdr in flv。

如果对enhance rtmp如何支持h265不了解，推荐详解Enhanced-RTMP支持H.265

1. enhance rtmp关于hdr

文档enhance-rtmp-v2.md中，metadata-frame章节规定相关支持HDR部分。

定义一个新的VideoPacketType.Metadata，其内容为AMF格式的metadata(AMF是一种简便的描述格式，非常节省内存)，具体内部有HDR相关的colorInfo数据(每次hdr参数更新的时候，就发送该metadata)。

type ColorInfo = {
  colorConfig: {
    // number of bits used to record the color channels for each pixel
    bitDepth:                 number, // SHOULD be 8, 10 or 12

    //
    // colorPrimaries, transferCharacteristics and matrixCoefficients are defined 
    // in ISO/IEC 23091-4/ITU-T H.273. The values are an index into 
    // respective tables which are described in "Colour primaries", 
    // "Transfer characteristics" and "Matrix coefficients" sections. 
    // It is RECOMMENDED to provide these values.
    //

    // indicates the chromaticity coordinates of the source color primaries
    colorPrimaries:           number, // enumeration [0-255]

    // opto-electronic transfer characteristic function (ex. PQ, HLG)
    transferCharacteristics:  number, // enumeration [0-255]

    // matrix coefficients used in deriving luma and chroma signals
    matrixCoefficients:       number, // enumeration [0-255]
  },

  hdrCll: {
    //
    // maximum value of the frame average light level
    // (in 1 cd/m2) of the entire playback sequence
    //
    maxFall:  number,     // [0.0001-10000]

    //
    // maximum light level of any single pixel (in 1 cd/m2)
    // of the entire playback sequence
    //
    maxCLL:   number,     // [0.0001-10000]
  },

  //
  // The hdrMdcv object defines mastering display (i.e., where
  // creative work is done during the mastering process) color volume (a.k.a., mdcv)
  // metadata which describes primaries, white point and min/max luminance. The
  // hdrMdcv object SHOULD be provided.
  //
  // Specification of the metadata along with its ranges adhere to the
  // ST 2086:2018 - SMPTE Standard (except for minLuminance see
  // comments below)
  //
  hdrMdcv: {
    //
    // Mastering display color volume (mdcv) xy Chromaticity Coordinates within CIE
    // 1931 color space.
    //
    // Values SHALL be specified with four decimal places. The x coordinate SHALL
    // be in the range [0.0001, 0.7400]. The y coordinate SHALL be 
    // in the range [0.0001, 0.8400].
    //
    redX:         number,
    redY:         number,
    greenX:       number,
    greenY:       number,
    blueX:        number,
    blueY:        number,
    whitePointX:  number,
    whitePointY:  number,

    //
    // max/min display luminance of the mastering display (in 1 cd/m2 ie. nits)
    //
    // note: ST 2086:2018 - SMPTE Standard specifies minimum display mastering
    // luminance in multiples of 0.0001 cd/m2.
    // 
    // For consistency we specify all values
    // in 1 cd/m2. Given that a hypothetical perfect screen has a peak brightness
    // of 10,000 nits and a black level of .0005 nits we do not need to
    // switch units to 0.0001 cd/m2 to increase resolution on the lower end of the
    // minLuminance property. The ranges (in nits) mentioned below suffice
    // the theoretical limit for Mastering Reference Displays and adhere to the
    // SMPTE ST 2084 standard (a.k.a., PQ) which is capable of representing full gamut
    // of luminance level.
    //
    maxLuminance: number,     // [5-10000]
    minLuminance: number,     // [0.0001-5]
  },
}

2. flv mux

在libavformat/flvenc.c中，新增flv_write_metadata_packet函数，用于添加metadata。

flv的数据结构:

typedef struct FLVContext {
    .....
    int metadata_pkt_written;//0: 还未写过，需要写；1: 写过了，不需要写。
} FLVContext;

在函数flv_write_metadata_packet中实现，见注释：

static void flv_write_metadata_packet(AVFormatContext *s, AVCodecParameters *par, unsigned int ts) {
    FLVContext *flv = s->priv_data;

    //不需要写入，就返回；可以通过设置这个标志变量来使能/去使能更新写metadata
    if (flv->metadata_pkt_written)
        return;
    //支持h265, av1, vp9
    if (par->codec_id == AV_CODEC_ID_HEVC || par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {
        ......
        //写入tag头，标识其为视频
        avio_w8(pb, FLV_TAG_TYPE_VIDEO); //write video tag type
        metadata_size_pos = avio_tell(pb);
        avio_wb24(pb, 0 + flags_size);
        put_timestamp(pb, ts); //ts = pkt->dts, gen
        avio_wb24(pb, flv->reserved);

        //根据enhance rtmp标志，写入FLV_IS_EX_HEADER标识，和fourCC的字段(hvc1 or av01 or vp09)
        if (par->codec_id == AV_CODEC_ID_HEVC) {
            avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD); // ExVideoTagHeader mode with PacketTypeMetadata
            avio_write(pb, "hvc1", 4);
        } else if (par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {
            avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD);
            avio_write(pb, par->codec_id == AV_CODEC_ID_AV1 ? "av01" : "vp09", 4);
        }
        //下面为写入AMF格式的hdr相关的colorInfo数据
        avio_w8(pb, AMF_DATA_TYPE_STRING);
        put_amf_string(pb, "colorInfo");

        avio_w8(pb, AMF_DATA_TYPE_OBJECT);

        put_amf_string(pb, "colorConfig");  // colorConfig

        avio_w8(pb, AMF_DATA_TYPE_OBJECT);

        if (par->color_trc != AVCOL_TRC_UNSPECIFIED &&
            par->color_trc < AVCOL_TRC_NB) {
            put_amf_string(pb, "transferCharacteristics");  // color_trc
            put_amf_double(pb, par->color_trc);
        }

        if (par->color_space != AVCOL_SPC_UNSPECIFIED &&
            par->color_space < AVCOL_SPC_NB) {
            put_amf_string(pb, "matrixCoefficients"); // colorspace
            put_amf_double(pb, par->color_space);
        }

        if (par->color_primaries != AVCOL_PRI_UNSPECIFIED &&
            par->color_primaries < AVCOL_PRI_NB) {
            put_amf_string(pb, "colorPrimaries"); // color_primaries
            put_amf_double(pb, par->color_primaries);
        }

        put_amf_string(pb, "");
        avio_w8(pb, AMF_END_OF_OBJECT);

        if (lightMetadata) {
            put_amf_string(pb, "hdrCll");
            avio_w8(pb, AMF_DATA_TYPE_OBJECT);

            put_amf_string(pb, "maxFall");
            put_amf_double(pb, lightMetadata->MaxFALL);

            put_amf_string(pb, "maxCLL");
            put_amf_double(pb, lightMetadata->MaxCLL);

            put_amf_string(pb, "");
            avio_w8(pb, AMF_END_OF_OBJECT);
        }

        if (displayMetadata && (displayMetadata->has_primaries || displayMetadata->has_luminance)) {
            put_amf_string(pb, "hdrMdcv");
            avio_w8(pb, AMF_DATA_TYPE_OBJECT);
            if (displayMetadata->has_primaries) {
                put_amf_string(pb, "redX");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][0]));

                put_amf_string(pb, "redY");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][1]));

                put_amf_string(pb, "greenX");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][0]));

                put_amf_string(pb, "greenY");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][1]));

                put_amf_string(pb, "blueX");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][0]));

                put_amf_string(pb, "blueY");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][1]));

                put_amf_string(pb, "whitePointX");
                put_amf_double(pb, av_q2d(displayMetadata->white_point[0]));

                put_amf_string(pb, "whitePointY");
                put_amf_double(pb, av_q2d(displayMetadata->white_point[1]));
            }
            if (displayMetadata->has_luminance) {
                put_amf_string(pb, "maxLuminance");
                put_amf_double(pb, av_q2d(displayMetadata->max_luminance));

                put_amf_string(pb, "minLuminance");
                put_amf_double(pb, av_q2d(displayMetadata->min_luminance));
            }
            put_amf_string(pb, "");
            avio_w8(pb, AMF_END_OF_OBJECT);
        }
        put_amf_string(pb, "");
        avio_w8(pb, AMF_END_OF_OBJECT);

        flv->metadata_pkt_written = 1;//标识写过了
    }
}

其中HDR的数据来源为AVCodecParameters *par数据结构中的内容：

typedef struct AVCodecParameters {
    ....
    /**
     * Video only. Additional colorspace characteristics.
     */
    enum AVColorRange                  color_range;
    enum AVColorPrimaries              color_primaries;
    enum AVColorTransferCharacteristic color_trc;
    enum AVColorSpace                  color_space;
    enum AVChromaLocation              chroma_location;

    /*
     * 类型: AV_PKT_DATA_CONTENT_LIGHT_LEVEL, 数据: AVContentLightMetadata* lightMetadata
     * 类型: AV_PKT_DATA_MASTERING_DISPLAY_METADATA, 数据: AVMasteringDisplayMetadata *displayMetadata
    */
    AVPacketSideData *coded_side_data;
    ....
}

3. flv demux

解析函数在libavformat/flvdec.c文件中，函数amf_parse_object中。

static int amf_parse_object(AVFormatContext *s, AVStream *astream,
                            AVStream *vstream, const char *key,
                            int64_t max_pos, int depth)
{
    FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;
    ......
    
    if (meta_video_color) {
        if (amf_type == AMF_DATA_TYPE_NUMBER ||
            amf_type == AMF_DATA_TYPE_BOOL) {
            if (!strcmp(key, "colorPrimaries")) {
                meta_video_color->primaries = num_val;
            } else if (!strcmp(key, "transferCharacteristics")) {
                meta_video_color->transfer_characteristics = num_val;
            } else if (!strcmp(key, "matrixCoefficients")) {
                meta_video_color->matrix_coefficients = num_val;
            } else if (!strcmp(key, "maxFall")) {
                meta_video_color->max_fall = num_val;
            } else if (!strcmp(key, "maxCLL")) {
                meta_video_color->max_cll = num_val;
            } else if (!strcmp(key, "redX")) {
                meta_video_color->mastering_meta.r_x = num_val;
            } else if (!strcmp(key, "redY")) {
                meta_video_color->mastering_meta.r_y = num_val;
            } else if (!strcmp(key, "greenX")) {
                meta_video_color->mastering_meta.g_x = num_val;
            } else if (!strcmp(key, "greenY")) {
                meta_video_color->mastering_meta.g_y = num_val;
            } else if (!strcmp(key, "blueX")) {
                meta_video_color->mastering_meta.b_x = num_val;
            } else if (!strcmp(key, "blueY")) {
                meta_video_color->mastering_meta.b_y = num_val;
            } else if (!strcmp(key, "whitePointX")) {
                meta_video_color->mastering_meta.white_x = num_val;
            } else if (!strcmp(key, "whitePointY")) {
                meta_video_color->mastering_meta.white_y = num_val;
            } else if (!strcmp(key, "maxLuminance")) {
                meta_video_color->mastering_meta.max_luminance = num_val;
            } else if (!strcmp(key, "minLuminance")) {
                meta_video_color->mastering_meta.min_luminance = num_val;
            }
        }
    }
    ......
}

4. 感谢Zhu Pengfei的提交

Author: Zhu Pengfei <411294962@qq.com>
Date:   Mon Mar 4 21:52:04 2024 +0800

    avformat/flvenc: support enhanced flv PacketTypeMetadata
    
    Signed-off-by: Zhu Pengfei <411294962@qq.com>
    Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
``