ffmpeg7.0 flv支持hdr

runner365.git2024-05-07 18:29

ffmpeg7.0 flv支持hdr

自从ffmpeg6.0应用enhance rtmp支持h265/av1的flv格式后，7.0迎来了flv的hdr能力。本文介绍ffmpeg7.0如何支持hdr in flv。

如果对enhance rtmp如何支持h265不了解，推荐详解Enhanced-RTMP支持H.265

1. enhance rtmp关于hdr

文档enhance-rtmp-v2.md中，metadata-frame章节规定相关支持HDR部分。

定义一个新的VideoPacketType.Metadata，其内容为AMF格式的metadata(AMF是一种简便的描述格式，非常节省内存)，具体内部有HDR相关的colorInfo数据(每次hdr参数更新的时候，就发送该metadata)。

复制代码 
type ColorInfo = {
  colorConfig: {
    // number of bits used to record the color channels for each pixel
    bitDepth:                 number, // SHOULD be 8, 10 or 12

    //
    // colorPrimaries, transferCharacteristics and matrixCoefficients are defined 
    // in ISO/IEC 23091-4/ITU-T H.273. The values are an index into 
    // respective tables which are described in "Colour primaries", 
    // "Transfer characteristics" and "Matrix coefficients" sections. 
    // It is RECOMMENDED to provide these values.
    //

    // indicates the chromaticity coordinates of the source color primaries
    colorPrimaries:           number, // enumeration [0-255]

    // opto-electronic transfer characteristic function (ex. PQ, HLG)
    transferCharacteristics:  number, // enumeration [0-255]

    // matrix coefficients used in deriving luma and chroma signals
    matrixCoefficients:       number, // enumeration [0-255]
  },

  hdrCll: {
    //
    // maximum value of the frame average light level
    // (in 1 cd/m2) of the entire playback sequence
    //
    maxFall:  number,     // [0.0001-10000]

    //
    // maximum light level of any single pixel (in 1 cd/m2)
    // of the entire playback sequence
    //
    maxCLL:   number,     // [0.0001-10000]
  },

  //
  // The hdrMdcv object defines mastering display (i.e., where
  // creative work is done during the mastering process) color volume (a.k.a., mdcv)
  // metadata which describes primaries, white point and min/max luminance. The
  // hdrMdcv object SHOULD be provided.
  //
  // Specification of the metadata along with its ranges adhere to the
  // ST 2086:2018 - SMPTE Standard (except for minLuminance see
  // comments below)
  //
  hdrMdcv: {
    //
    // Mastering display color volume (mdcv) xy Chromaticity Coordinates within CIE
    // 1931 color space.
    //
    // Values SHALL be specified with four decimal places. The x coordinate SHALL
    // be in the range [0.0001, 0.7400]. The y coordinate SHALL be 
    // in the range [0.0001, 0.8400].
    //
    redX:         number,
    redY:         number,
    greenX:       number,
    greenY:       number,
    blueX:        number,
    blueY:        number,
    whitePointX:  number,
    whitePointY:  number,

    //
    // max/min display luminance of the mastering display (in 1 cd/m2 ie. nits)
    //
    // note: ST 2086:2018 - SMPTE Standard specifies minimum display mastering
    // luminance in multiples of 0.0001 cd/m2.
    // 
    // For consistency we specify all values
    // in 1 cd/m2. Given that a hypothetical perfect screen has a peak brightness
    // of 10,000 nits and a black level of .0005 nits we do not need to
    // switch units to 0.0001 cd/m2 to increase resolution on the lower end of the
    // minLuminance property. The ranges (in nits) mentioned below suffice
    // the theoretical limit for Mastering Reference Displays and adhere to the
    // SMPTE ST 2084 standard (a.k.a., PQ) which is capable of representing full gamut
    // of luminance level.
    //
    maxLuminance: number,     // [5-10000]
    minLuminance: number,     // [0.0001-5]
  },
}

2. flv mux

在libavformat/flvenc.c中，新增flv_write_metadata_packet函数，用于添加metadata。

flv的数据结构:

复制代码 
typedef struct FLVContext {
    .....
    int metadata_pkt_written;//0: 还未写过，需要写；1: 写过了，不需要写。
} FLVContext;

在函数flv_write_metadata_packet中实现，见注释：

复制代码 
static void flv_write_metadata_packet(AVFormatContext *s, AVCodecParameters *par, unsigned int ts) {
    FLVContext *flv = s->priv_data;

    //不需要写入，就返回；可以通过设置这个标志变量来使能/去使能更新写metadata
    if (flv->metadata_pkt_written)
        return;
    //支持h265, av1, vp9
    if (par->codec_id == AV_CODEC_ID_HEVC || par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {
        ......
        //写入tag头，标识其为视频
        avio_w8(pb, FLV_TAG_TYPE_VIDEO); //write video tag type
        metadata_size_pos = avio_tell(pb);
        avio_wb24(pb, 0 + flags_size);
        put_timestamp(pb, ts); //ts = pkt->dts, gen
        avio_wb24(pb, flv->reserved);

        //根据enhance rtmp标志，写入FLV_IS_EX_HEADER标识，和fourCC的字段(hvc1 or av01 or vp09)
        if (par->codec_id == AV_CODEC_ID_HEVC) {
            avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD); // ExVideoTagHeader mode with PacketTypeMetadata
            avio_write(pb, "hvc1", 4);
        } else if (par->codec_id == AV_CODEC_ID_AV1 || par->codec_id == AV_CODEC_ID_VP9) {
            avio_w8(pb, FLV_IS_EX_HEADER | PacketTypeMetadata| FLV_FRAME_VIDEO_INFO_CMD);
            avio_write(pb, par->codec_id == AV_CODEC_ID_AV1 ? "av01" : "vp09", 4);
        }
        //下面为写入AMF格式的hdr相关的colorInfo数据
        avio_w8(pb, AMF_DATA_TYPE_STRING);
        put_amf_string(pb, "colorInfo");

        avio_w8(pb, AMF_DATA_TYPE_OBJECT);

        put_amf_string(pb, "colorConfig");  // colorConfig

        avio_w8(pb, AMF_DATA_TYPE_OBJECT);

        if (par->color_trc != AVCOL_TRC_UNSPECIFIED &&
            par->color_trc < AVCOL_TRC_NB) {
            put_amf_string(pb, "transferCharacteristics");  // color_trc
            put_amf_double(pb, par->color_trc);
        }

        if (par->color_space != AVCOL_SPC_UNSPECIFIED &&
            par->color_space < AVCOL_SPC_NB) {
            put_amf_string(pb, "matrixCoefficients"); // colorspace
            put_amf_double(pb, par->color_space);
        }

        if (par->color_primaries != AVCOL_PRI_UNSPECIFIED &&
            par->color_primaries < AVCOL_PRI_NB) {
            put_amf_string(pb, "colorPrimaries"); // color_primaries
            put_amf_double(pb, par->color_primaries);
        }

        put_amf_string(pb, "");
        avio_w8(pb, AMF_END_OF_OBJECT);

        if (lightMetadata) {
            put_amf_string(pb, "hdrCll");
            avio_w8(pb, AMF_DATA_TYPE_OBJECT);

            put_amf_string(pb, "maxFall");
            put_amf_double(pb, lightMetadata->MaxFALL);

            put_amf_string(pb, "maxCLL");
            put_amf_double(pb, lightMetadata->MaxCLL);

            put_amf_string(pb, "");
            avio_w8(pb, AMF_END_OF_OBJECT);
        }

        if (displayMetadata && (displayMetadata->has_primaries || displayMetadata->has_luminance)) {
            put_amf_string(pb, "hdrMdcv");
            avio_w8(pb, AMF_DATA_TYPE_OBJECT);
            if (displayMetadata->has_primaries) {
                put_amf_string(pb, "redX");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][0]));

                put_amf_string(pb, "redY");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[0][1]));

                put_amf_string(pb, "greenX");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][0]));

                put_amf_string(pb, "greenY");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[1][1]));

                put_amf_string(pb, "blueX");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][0]));

                put_amf_string(pb, "blueY");
                put_amf_double(pb, av_q2d(displayMetadata->display_primaries[2][1]));

                put_amf_string(pb, "whitePointX");
                put_amf_double(pb, av_q2d(displayMetadata->white_point[0]));

                put_amf_string(pb, "whitePointY");
                put_amf_double(pb, av_q2d(displayMetadata->white_point[1]));
            }
            if (displayMetadata->has_luminance) {
                put_amf_string(pb, "maxLuminance");
                put_amf_double(pb, av_q2d(displayMetadata->max_luminance));

                put_amf_string(pb, "minLuminance");
                put_amf_double(pb, av_q2d(displayMetadata->min_luminance));
            }
            put_amf_string(pb, "");
            avio_w8(pb, AMF_END_OF_OBJECT);
        }
        put_amf_string(pb, "");
        avio_w8(pb, AMF_END_OF_OBJECT);

        flv->metadata_pkt_written = 1;//标识写过了
    }
}

其中HDR的数据来源为AVCodecParameters *par数据结构中的内容：

复制代码 
typedef struct AVCodecParameters {
    ....
    /**
     * Video only. Additional colorspace characteristics.
     */
    enum AVColorRange                  color_range;
    enum AVColorPrimaries              color_primaries;
    enum AVColorTransferCharacteristic color_trc;
    enum AVColorSpace                  color_space;
    enum AVChromaLocation              chroma_location;

    /*
     * 类型: AV_PKT_DATA_CONTENT_LIGHT_LEVEL, 数据: AVContentLightMetadata* lightMetadata
     * 类型: AV_PKT_DATA_MASTERING_DISPLAY_METADATA, 数据: AVMasteringDisplayMetadata *displayMetadata
    */
    AVPacketSideData *coded_side_data;
    ....
}

3. flv demux

解析函数在libavformat/flvdec.c文件中，函数amf_parse_object中。

复制代码 
static int amf_parse_object(AVFormatContext *s, AVStream *astream,
                            AVStream *vstream, const char *key,
                            int64_t max_pos, int depth)
{
    FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;
    ......
    
    if (meta_video_color) {
        if (amf_type == AMF_DATA_TYPE_NUMBER ||
            amf_type == AMF_DATA_TYPE_BOOL) {
            if (!strcmp(key, "colorPrimaries")) {
                meta_video_color->primaries = num_val;
            } else if (!strcmp(key, "transferCharacteristics")) {
                meta_video_color->transfer_characteristics = num_val;
            } else if (!strcmp(key, "matrixCoefficients")) {
                meta_video_color->matrix_coefficients = num_val;
            } else if (!strcmp(key, "maxFall")) {
                meta_video_color->max_fall = num_val;
            } else if (!strcmp(key, "maxCLL")) {
                meta_video_color->max_cll = num_val;
            } else if (!strcmp(key, "redX")) {
                meta_video_color->mastering_meta.r_x = num_val;
            } else if (!strcmp(key, "redY")) {
                meta_video_color->mastering_meta.r_y = num_val;
            } else if (!strcmp(key, "greenX")) {
                meta_video_color->mastering_meta.g_x = num_val;
            } else if (!strcmp(key, "greenY")) {
                meta_video_color->mastering_meta.g_y = num_val;
            } else if (!strcmp(key, "blueX")) {
                meta_video_color->mastering_meta.b_x = num_val;
            } else if (!strcmp(key, "blueY")) {
                meta_video_color->mastering_meta.b_y = num_val;
            } else if (!strcmp(key, "whitePointX")) {
                meta_video_color->mastering_meta.white_x = num_val;
            } else if (!strcmp(key, "whitePointY")) {
                meta_video_color->mastering_meta.white_y = num_val;
            } else if (!strcmp(key, "maxLuminance")) {
                meta_video_color->mastering_meta.max_luminance = num_val;
            } else if (!strcmp(key, "minLuminance")) {
                meta_video_color->mastering_meta.min_luminance = num_val;
            }
        }
    }
    ......
}

4. 感谢Zhu Pengfei的提交

复制代码 
Author: Zhu Pengfei <411294962@qq.com>
Date:   Mon Mar 4 21:52:04 2024 +0800

    avformat/flvenc: support enhanced flv PacketTypeMetadata
    
    Signed-off-by: Zhu Pengfei <411294962@qq.com>
    Signed-off-by: Steven Liu <lq@chinaffmpeg.org>
``
相关推荐
xcLeigh12 小时前
HTML5实现好看的视频播放器(三种风格，附源码)
前端·音视频·html5
骄傲的心别枯萎15 小时前
RV1126 NO.57：ROCKX+RV1126人脸识别推流项目之读取人脸图片并把特征值保存到sqlite3数据库
数据库·opencv·计算机视觉·sqlite·音视频·rv1126
好游科技16 小时前
IM即时通讯系统：安全可控、功能全面的社交解决方案全解析
安全·音视频·webrtc·im即时通讯·私有化部署im即时通讯·社交app
EasyDSS17 小时前
视频直播点播平台EasyDSS构建高并发、低延迟的远程教学直播新模式
音视频
GIOTTO情17 小时前
多模态舆情监测技术深度解析：Infoseek 如何实现 AI 造假与短视频舆情的精准捕捉？
人工智能·音视频
音视频牛哥17 小时前
C# 开发工业级 RTSP/RTMP 播放器实战：基于 SmartMediakit 的低延迟与高可靠性设计
音视频·rtsp播放器·rtmp播放器·windows rtsp播放器·windows rtmp播放器·c# rtsp播放器·c# rtmp播放器
JellyDDD17 小时前
【悬赏】Android WebRTC 数字人项目回声问题排查（AEC / AudioMode）
音视频·webrtc
于是我说18 小时前
如何判断一个视频到底是真实 MP4 直链，还是流媒体M3U8
网络·音视频
gf132111118 小时前
剪映草稿位置坐标换算
音视频
ACP广源盛1392462567318 小时前
GSV1011@ACP#1011产品规格详解及产品应用分享
嵌入式硬件·计算机外设·音视频
热门推荐
01GitHub 镜像站点02UV安装并设置国内源03Linux下V2Ray安装配置指南04RedissonClient的配置解析05安娜的档案(Anna’s Archive) 镜像网站/国内最新可访问入口（持续更新）06在VSCode配置Java开发环境的保姆级教程（适配各类AI编程IDE）07BongoCat - 跨平台键盘猫动画工具08Open-AutoGLM Windows 安装部署教程09jdk21下载、安装（Windows、Linux、macOS）10【AutoGLM部署】本地私有化部署AI手机Agent