iOS使用Metal对采集视频进行渲染

Unlimitedz2025-05-25 14:55

首先对于每一帧的采集成功后，都会有一个闭包回调，然后调用renderPixelBuffer

再看renderPixelBuffer的实现

会调用draw方法

swift 复制代码 
// MARK: - MTKViewDelegate
extension KFMetalView: MTKViewDelegate {
    func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
        viewportSize = vector_uint2(UInt32(size.width), UInt32(size.height))
    }
    
    func draw(in view: MTKView) {
        // Metal 视图回调，有数据情况下渲染视图
        weak var weakSelf = self
        renderQueue.async {
            guard let self = weakSelf else { return }
            self.drawInMTKView(view)
        }
    }
}

最后看向真正的实现：drawInMTKView

swift 复制代码 
/// 渲染数据
    private func drawInMTKView(_ view: MTKView) {
        semaphore.wait()
        defer { semaphore.signal() }
        
        guard let pixelBuffer = self.pixelBuffer else { 
            return 
        }
        self.pixelBuffer = nil
        
        // 检查并初始化命令队列
        if commandQueue == nil {
            guard let device = mtkView.device else {
                return
            }
            commandQueue = device.makeCommandQueue()
        }
        
        guard let commandQueue = self.commandQueue,
              let renderPassDescriptor = view.currentRenderPassDescriptor,
              let drawable = view.currentDrawable else { 
            return 
        }
        
        let commandBuffer = commandQueue.makeCommandBuffer()
        
        guard let renderEncoder = commandBuffer?.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { 
            return 
        }
        
        renderEncoder.setViewport(MTLViewport(originX: 0.0, originY: 0.0,
                                             width: Double(viewportSize.x),
                                             height: Double(viewportSize.y),
                                             znear: -1.0, zfar: 1.0))
        
        let isRenderYUV = CVPixelBufferGetPlaneCount(pixelBuffer) > 1
        
        // 确保创建管道状态
        if pipelineState == nil {
            setupPipeline(isYUV: isRenderYUV)
        }
        
        guard let pipelineState = pipelineState else {
            renderEncoder.endEncoding()
            return
        }
        
        renderEncoder.setRenderPipelineState(pipelineState)
        
        
        
        
        
        if updateFillMode {
            updateVertices()
            updateFillMode = false
        }
        
        if let vertices = vertices {
            // 使用常量值0作为顶点缓冲索引，与Metal着色器中定义一致
            renderEncoder.setVertexBuffer(vertices, offset: 0, index: 0)
        } else {
            renderEncoder.endEncoding()
            return
        }
        
        if isRenderYUV {
            // 处理 YUV 纹理
            var textureY: MTLTexture?
            var textureUV: MTLTexture?
            
            if let textureCache = textureCache {
                // 确保pixelBuffer被锁定以便安全访问
                CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
                defer {
                    CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
                }
                
                let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)
                let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)
                let pixelFormat = MTLPixelFormat.r8Unorm
                
                var cvTextureY: CVMetalTexture?
                let statusY = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureY)
                
                if statusY == kCVReturnSuccess, let cvTextureY = cvTextureY {
                    textureY = CVMetalTextureGetTexture(cvTextureY)
                }
                
                let widthUV = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1)
                let heightUV = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1)
                let pixelFormatUV = MTLPixelFormat.rg8Unorm
                
                var cvTextureUV: CVMetalTexture?
                let statusUV = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormatUV, widthUV, heightUV, 1, &cvTextureUV)
                
                if statusUV == kCVReturnSuccess, let cvTextureUV = cvTextureUV {
                    textureUV = CVMetalTextureGetTexture(cvTextureUV)
                }
            }
            
            if let textureY = textureY, let textureUV = textureUV {
                // 设置纹理，使用shader中定义的索引
                renderEncoder.setFragmentTexture(textureY, index: 0)
                renderEncoder.setFragmentTexture(textureUV, index: 1)
            } else {
                renderEncoder.endEncoding()
                return
            }
            
            if yuvMatrix == nil {
                // 获取颜色空间信息，如果没有就使用默认值
                let colorSpace: CFTypeRef
                let isFullRange: Bool
                
                // 尝试从pixelBuffer获取YUV颜色空间信息
                if let matrixKey = CVBufferCopyAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, nil) {
                    colorSpace = matrixKey
                    isFullRange = pixelBufferIsFullRange(pixelBuffer)
                } else {
                    // 如果pixelBuffer中没有颜色空间信息，使用默认值
                    // 对于相机视频，通常使用BT.601标准和full range
                    colorSpace = kCVImageBufferYCbCrMatrix_ITU_R_601_4
                    
                    // 检查格式类型判断是否为full range
                    let format = CVPixelBufferGetPixelFormatType(pixelBuffer)
                    isFullRange = (format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
                }
                
                setupYUVMatrix(isFullRange: isFullRange, colorSpace: colorSpace)
            }
            
            // 设置矩阵缓冲区，使用shader中定义的索引
            if let yuvMatrix = yuvMatrix {
                renderEncoder.setFragmentBuffer(yuvMatrix, offset: 0, index: 0)
            }
            
        } else {
            // 处理 RGB 纹理
            var textureRGB: MTLTexture?
            
            if let textureCache = textureCache {
                let width = CVPixelBufferGetWidth(pixelBuffer)
                let height = CVPixelBufferGetHeight(pixelBuffer)
                let pixelFormat = MTLPixelFormat.bgra8Unorm
                
                var cvTextureRGB: CVMetalTexture?
                let status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureRGB)
                
                if status == kCVReturnSuccess, let cvTextureRGB = cvTextureRGB {
                    textureRGB = CVMetalTextureGetTexture(cvTextureRGB)
                }
            }
            
            if let textureRGB = textureRGB {
                // 设置RGB纹理，使用shader中定义的索引
                renderEncoder.setFragmentTexture(textureRGB, index: 0)
            } else {
                renderEncoder.endEncoding()
                return
            }
        }
        
        renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: numVertices)
        renderEncoder.endEncoding()
        
        commandBuffer?.present(drawable)
        commandBuffer?.commit()
    }

drawInMTKView 函数的实现过程总结：

  1. 获取并清除像素缓冲区

    • 使用信号量确保线程安全访问
    • 获取当前的 pixelBuffer 并立即清除引用

  2. 准备渲染资源

    • 检查并初始化命令队列
    • 获取当前渲染描述符和可绘制对象
    • 创建命令缓冲区和渲染编码器

  3. 设置渲染状态

    • 设置视口大小
    • 检测是否为 YUV 格式（通过平面数量判断）
    • 确保渲染管道状态已创建（如未创建则调用 setupPipeline）
    • 检查顶点数据是否需要更新（根据填充模式）

  4. 设置顶点缓冲区

    • 将顶点数据传递给渲染编码器

  5. 处理纹理

    • 对于 YUV 格式：
      • 创建 Y 平面和 UV 平面的纹理
      • 设置 YUV 转换矩阵（如果尚未设置）
      • 将纹理和矩阵传递给片段着色器
    • 对于 RGB 格式：
      • 创建单一 RGB 纹理并传递给片段着色器

  6. 执行渲染

    • 绘制三角形带（triangleStrip）
    • 结束编码
    • 提交命令缓冲区并呈现到屏幕

  7. 错误处理

    • 在各个步骤中检查资源是否正确创建，如果失败则提前结束渲染过程

整个过程实现了从 CVPixelBuffer（可能是 YUV 或 RGB 格式）到 Metal 渲染的完整流程，包括纹理创建、格式转换和根据填充模式调整显示比例。

对于渲染的其他部分，是基于渲染管线进行渲染的，其他操作和渲染三角形之类的大差不差，比如配置Metal文件，创建渲染管线等。

swift 复制代码 
//
//  Shader.metal
//  VideoDemo
//
//  Created by ricard.li on 2025/5/23.
//

#include <metal_stdlib>
using namespace metal;

typedef struct {
    // 顶点坐标，4 维向量。
    vector_float4 position;
    // 纹理坐标。
    vector_float2 textureCoordinate;
} KFVertex;


typedef struct {
    // YUV 矩阵，使用float4x4替代float3x3以确保内存对齐
    float4x4 matrix;
    // 是否为 full range，使用uint代替bool以确保内存布局一致
    uint fullRange;
} KFConvertMatrix;


// 顶点的桥接枚举值 KFVertexInputIndexVertices。
typedef enum KFVertexInputIndex {
    KFVertexInputIndexVertices = 0,
} KFVertexInputIndex;


// YUV 矩阵的桥接枚举值 KFFragmentInputIndexMatrix。
typedef enum KFFragmentBufferIndex {
    KFFragmentInputIndexMatrix = 0,
} KFMetalFragmentBufferIndex;


// YUV 数据的桥接枚举值 KFFragmentTextureIndexTextureY、KFFragmentTextureIndexTextureUV。
typedef enum KFFragmentYUVTextureIndex {
    KFFragmentTextureIndexTextureY = 0,
    KFFragmentTextureIndexTextureUV = 1,
} KFFragmentYUVTextureIndex;


// RGBA 数据的桥接枚举值 KFFragmentTextureIndexTextureRGB。
typedef enum KFFragmentRGBTextureIndex {
    KFFragmentTextureIndexTextureRGB = 0,
} KFFragmentRGBTextureIndex;

// 定义了一个类型为 RasterizerData 的结构体，里面有一个 float4 向量和 float2 向量。
typedef struct {
    // float4：4 维向量；
    // clipSpacePosition：参数名，表示顶点；
    // [[position]]：position 是顶点修饰符，这是苹果内置的语法，不能改变，表示顶点信息。
    float4 clipSpacePosition [[position]];
    // float2：2 维向量；
    // textureCoordinate：参数名，这里表示纹理。
    float2 textureCoordinate;
} RasterizerData;

// 顶点函数通过一个自定义的结构体，返回对应的数据；顶点函数的输入参数也可以是自定义结构体。

// 顶点函数
// vertex：函数修饰符，表示顶点函数；
// RasterizerData：返回值类型；
// vertexShader：函数名；
// [[vertex_id]]：vertex_id 是顶点 id 修饰符，苹果内置的语法不可改变；
// [[buffer(YYImageVertexInputIndexVertexs)]]：buffer 是缓存数据修饰符，苹果内置的语法不可改变，YYImageVertexInputIndexVertexs 是索引；
// constant：是变量类型修饰符，表示存储在 device 区域。
vertex RasterizerData vertexShader(uint vertexID [[vertex_id]],
                                   constant KFVertex *vertexArray [[buffer(KFVertexInputIndexVertices)]]) {
    RasterizerData out;
    out.clipSpacePosition = vertexArray[vertexID].position;
    out.textureCoordinate = vertexArray[vertexID].textureCoordinate;
    
    return out;
}

// 片元函数
// fragment：函数修饰符，表示片元函数；
// float4：返回值类型，返回 RGBA；
// fragmentImageShader：函数名；
// RasterizerData：参数类型；
// input：变量名；
// [[stage_in]：stage_in 表示这个数据来自光栅化，光栅化是顶点处理之后的步骤，业务层无法修改。
// texture2d：类型表示纹理；
// textureY：表示 Y 通道；
// textureUV：表示 UV 通道；
// [[texture(index)]]：纹理修饰符；可以加索引：[[texture(0)]] 对应纹理 0，[[texture(1)]] 对应纹理 1；
// KFFragmentTextureIndexTextureY、KFFragmentTextureIndexTextureUV：表示纹理索引。
fragment float4 yuvSamplingShader(RasterizerData input [[stage_in]],
                                  texture2d<float> textureY [[texture(KFFragmentTextureIndexTextureY)]],
                                  texture2d<float> textureUV [[texture(KFFragmentTextureIndexTextureUV)]],
                                  constant KFConvertMatrix *convertMatrix [[buffer(KFFragmentInputIndexMatrix)]]) {
    constexpr sampler textureSampler (mag_filter::linear, min_filter::linear);
    
    // 初始化YUV向量
    float3 yuv;
    
    // 根据范围处理Y值
    if (convertMatrix->fullRange != 0) { // full range.
        yuv.x = textureY.sample(textureSampler, input.textureCoordinate).r;
    } else { // video range.
        yuv.x = textureY.sample(textureSampler, input.textureCoordinate).r - (16.0 / 255.0);
    }
    
    // 获取UV值，并从[0,1]范围转换到[-0.5,0.5]
    yuv.yz = textureUV.sample(textureSampler, input.textureCoordinate).rg - 0.5;
    
    // 使用YUV转RGB矩阵计算RGB值 - 使用float4x4矩阵，但仍提取float3结果
    float3 rgb = float3(convertMatrix->matrix * float4(yuv, 1.0));
    
    // 确保结果在合法范围内
    rgb = clamp(rgb, 0.0, 1.0);
  
    return float4(rgb, 1.0);
}

// 片元函数
// fragment：函数修饰符，表示片元函数；
// float4：返回值类型，返回 RGBA；
// fragmentImageShader：函数名；
// RasterizerData：参数类型；
// input：变量名；
// [[stage_in]]：stage_in 表示这个数据来自光栅化，光栅化是顶点处理之后的步骤，业务层无法修改。
// texture2d：类型表示纹理；
// colorTexture：代表 RGBA 数据；
// [[texture(index)]]：纹理修饰符；可以加索引：[[texture(0)]] 对应纹理 0，[[texture(1)]] 对应纹理 1；
// KFFragmentTextureIndexTextureRGB：表示纹理索引。
fragment float4 rgbSamplingShader(RasterizerData input [[stage_in]],
                                  texture2d<half> colorTexture [[texture(KFFragmentTextureIndexTextureRGB)]]) {
    constexpr sampler textureSampler (mag_filter::linear, min_filter::linear);
    
    half4 colorSample = colorTexture.sample(textureSampler, input.textureCoordinate);
  
    return float4(colorSample);
}
swift 复制代码 
//
//  KFMetalView.swift
//  VideoDemo
//
//  Created by ricard.li on 2025/5/23.
//

import UIKit
import MetalKit
import AVFoundation
import MetalPerformanceShaders
import simd

// 渲染画面填充模式
enum KFMetalViewContentMode: Int {
    case stretch = 0  // 自动填充满，可能会变形
    case fit = 1      // 按比例适配，可能会有黑边
    case fill = 2     // 根据比例裁剪后填充满
}

// 颜色空间转换矩阵，BT.601 Video Range
private let kFColorMatrix601VideoRange = matrix_float4x4(
    simd_float4(1.164, 1.164, 1.164, 0.0),
    simd_float4(0.0, -0.392, 2.017, 0.0),
    simd_float4(1.596, -0.813, 0.0, 0.0),
    simd_float4(0.0, 0.0, 0.0, 1.0)
)

// 颜色空间转换矩阵，BT.601 Full Range
private let kFColorMatrix601FullRange = matrix_float4x4(
    simd_float4(1.0, 1.0, 1.0, 0.0),
    simd_float4(0.0, -0.343, 1.765, 0.0),
    simd_float4(1.4, -0.711, 0.0, 0.0),
    simd_float4(0.0, 0.0, 0.0, 1.0)
)

// 颜色空间转换矩阵，BT.709 Video Range
private let kFColorMatrix709VideoRange = matrix_float4x4(
    simd_float4(1.164, 1.164, 1.164, 0.0),
    simd_float4(0.0, -0.213, 2.112, 0.0),
    simd_float4(1.793, -0.533, 0.0, 0.0),
    simd_float4(0.0, 0.0, 0.0, 1.0)
)

// 颜色空间转换矩阵，BT.709 Full Range
private let kFColorMatrix709FullRange = matrix_float4x4(
    simd_float4(1.0, 1.0, 1.0, 0.0),
    simd_float4(0.0, -0.187, 1.856, 0.0),
    simd_float4(1.575, -0.468, 0.0, 0.0),
    simd_float4(0.0, 0.0, 0.0, 1.0)
)



class KFMetalView: UIView {
    // MARK: - Public Properties
    
    // 画面填充模式
    var fillMode: KFMetalViewContentMode = .fit {
        didSet {
            updateFillMode = true
        }
    }
    
    // MARK: - Private Properties
    
    // 外层输入的最后一帧数据
    private var pixelBuffer: CVPixelBuffer?
    // 处理 PixelBuffer 锁，防止外层输入线程与渲染线程同时操作 Crash
    private let semaphore = DispatchSemaphore(value: 1)
    // 纹理缓存，根据 pixelbuffer 获取纹理
    private var textureCache: CVMetalTextureCache?
    // Metal 渲染的 view
    private var mtkView: MTKView!
    // 视口大小
    private var viewportSize = vector_uint2(0, 0)
    // 渲染管道，管理顶点函数和片元函数
    private var pipelineState: MTLRenderPipelineState?
    // 渲染指令队列
    private var commandQueue: MTLCommandQueue?
    // 顶点缓存对象
    private var vertices: MTLBuffer?
    // 顶点数量
    private var numVertices: Int = 0
    // YUV 数据矩阵对象
    private var yuvMatrix: MTLBuffer?
    // 填充模式变更标记
    private var updateFillMode = true
    // pixelBuffer 数据尺寸
    private var pixelBufferSize = CGSize.zero
    // 当前视图大小
    private var currentViewSize = CGSize.zero
    // 渲染线程
    private let renderQueue = DispatchQueue(label: "com.KeyFrameKit.metalView.renderQueue", qos: .userInteractive)
    
    // MARK: - Lifecycle
    
    override init(frame: CGRect) {
        super.init(frame: frame)
        setupView(frame: frame)
    }
    
    required init?(coder: NSCoder) {
        super.init(coder: coder)
        setupView(frame: bounds)
    }
    
    private func setupView(frame: CGRect) {
        currentViewSize = frame.size
        fillMode = .fit
        updateFillMode = true
        
        // 设置视图背景色为黑色
        backgroundColor = .black
        
        // 创建 Metal 渲染视图且添加到当前视图
        mtkView = MTKView(frame: bounds)
        guard let device = MTLCreateSystemDefaultDevice() else {
            return 
        }
        
        mtkView.device = device
        mtkView.backgroundColor = .black  // 设置Metal视图背景色为黑色
        // 确保使用正确的像素格式
        mtkView.colorPixelFormat = .bgra8Unorm
        // 确保每次绘制都会调用delegate
        mtkView.isPaused = false
        // 确保绘制能够连续进行
        mtkView.enableSetNeedsDisplay = false
        
        addSubview(mtkView)
        mtkView.delegate = self
        mtkView.framebufferOnly = true
        viewportSize = vector_uint2(UInt32(mtkView.drawableSize.width), UInt32(mtkView.drawableSize.height))
        
        // 创建纹理缓存
        var newTextureCache: CVMetalTextureCache?
        let status = CVMetalTextureCacheCreate(nil, nil, device, nil, &newTextureCache)
        if status == kCVReturnSuccess {
            textureCache = newTextureCache
        }
        
        // 预先创建命令队列
        commandQueue = device.makeCommandQueue()
    }
    //确保Metal视图(mtkView)与包含它的父视图(KFMetalView)大小完全一致   
    override func layoutSubviews() {
        // 视图自动调整布局，同步至 Metal 视图
        super.layoutSubviews()
        mtkView.frame = bounds
        currentViewSize = bounds.size
    }
    
    deinit {
        // 释放最后一帧数据、纹理缓存
        semaphore.wait()
        pixelBuffer = nil
        
        if let textureCache = textureCache {
            CVMetalTextureCacheFlush(textureCache, 0)
        }
        textureCache = nil
        semaphore.signal()
        
        mtkView.releaseDrawables()
    }
    
    // MARK: - Public Methods
    
    /// 渲染 CVPixelBuffer
    func renderPixelBuffer(_ pixelBuffer: CVPixelBuffer) {
        semaphore.wait()
        self.pixelBuffer = pixelBuffer
        pixelBufferSize = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))
        semaphore.signal()
        
        // 确保在主线程调用setNeedsDisplay触发绘制
        DispatchQueue.main.async { [weak self] in
            self?.mtkView.draw()
        }
    }
    
    // MARK: - Private Methods
    
    /// 初始化渲染管道
    private func setupPipeline(isYUV: Bool) {
        guard let device = mtkView.device else { 
            return 
        }
        
        // 获取默认库
        guard let library = device.makeDefaultLibrary() else {
            return
        }
        
        // 根据是否处理YUV选择顶点和片元着色器
        let vertexFunctionName = "vertexShader"
        let fragmentFunctionName = isYUV ? "yuvSamplingShader" : "rgbSamplingShader"
        
        guard let vertexFunction = library.makeFunction(name: vertexFunctionName) else {
            return
        }
        
        guard let fragmentFunction = library.makeFunction(name: fragmentFunctionName) else {
            return
        }
        
        let pipelineStateDescriptor = MTLRenderPipelineDescriptor()
        pipelineStateDescriptor.vertexFunction = vertexFunction
        pipelineStateDescriptor.fragmentFunction = fragmentFunction
        pipelineStateDescriptor.colorAttachments[0].pixelFormat = mtkView.colorPixelFormat
        
        do {
            pipelineState = try device.makeRenderPipelineState(descriptor: pipelineStateDescriptor)
            if commandQueue == nil {
                commandQueue = device.makeCommandQueue()
            }
        } catch {
            // 使用do-catch而不是print，避免不必要的日志输出
        }
    }
    
    /// 初始化 YUV 矩阵
    private func setupYUVMatrix(isFullRange: Bool, colorSpace: CFTypeRef) {
        guard let device = mtkView.device else { 
            return 
        }
        
        // 选择正确的YUV转RGB矩阵
        var matrix: matrix_float4x4
        
        // 安全地检查颜色空间类型
        let colorSpaceString = CFCopyDescription(colorSpace) as String
        let is601 = colorSpaceString.contains("601")
        let is709 = colorSpaceString.contains("709")
        
        if is601 {
            if isFullRange {
                matrix = kFColorMatrix601FullRange
            } else {
                matrix = kFColorMatrix601VideoRange
            }
        } else if is709 {
            if isFullRange {
                matrix = kFColorMatrix709FullRange
            } else {
                matrix = kFColorMatrix709VideoRange
            }
        } else {
            // 默认使用BT.601
            if isFullRange {
                matrix = kFColorMatrix601FullRange
            } else {
                matrix = kFColorMatrix601VideoRange
            }
        }
        
        // 创建转换矩阵结构体，将Bool转换为UInt32
        var convertMatrix = KFConvertMatrix(matrix: matrix, fullRange: isFullRange ? 1 : 0)
        
        // 创建矩阵缓冲区
        yuvMatrix = device.makeBuffer(bytes: &convertMatrix, length: MemoryLayout<KFConvertMatrix>.size, options: .storageModeShared)
    }
    
    /// 更新顶点数据
    private func updateVertices() {
        guard let device = mtkView.device else { return }
        
        var widthScaling: Float = 1.0
        var heightScaling: Float = 1.0
        
        if !currentViewSize.equalTo(.zero) && !pixelBufferSize.equalTo(.zero) {
            // 计算视图和纹理的宽高比
            let viewAspect = Float(currentViewSize.width / currentViewSize.height)
            let textureAspect = Float(pixelBufferSize.width / pixelBufferSize.height)
            
            switch fillMode {
            case .stretch:
                // 拉伸模式，直接使用 1.0
                widthScaling = 1.0
                heightScaling = 1.0
                
            case .fit:
                // 适配模式，保持比例，可能有黑边
                if textureAspect > viewAspect {
                    // 视频比例比视图宽，高度需要缩小
                    widthScaling = 1.0
                    heightScaling = viewAspect / textureAspect
                } else {
                    // 视频比例比视图窄，宽度需要缩小
                    widthScaling = textureAspect / viewAspect
                    heightScaling = 1.0
                }
                
            case .fill:
                // 填充模式，确保没有黑边，可能会裁剪部分内容
                if textureAspect > viewAspect {
                    // 视频比例比视图宽，宽度需要裁剪
                    widthScaling = viewAspect / textureAspect
                    heightScaling = 1.0
                } else {
                    // 视频比例比视图窄，高度需要裁剪
                    widthScaling = 1.0
                    heightScaling = textureAspect / viewAspect
                }
                
                // 填充模式要确保覆盖整个屏幕，所以这里要取倒数
                widthScaling = 1.0 / widthScaling
                heightScaling = 1.0 / heightScaling
            }
        }
        
        // 创建顶点数组，与shader中的KFVertex结构一致
        let quadVertices: [KFVertex] = [
            KFVertex(position: vector_float4(-widthScaling, -heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(0.0, 1.0)),
            KFVertex(position: vector_float4(widthScaling, -heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(1.0, 1.0)),
            KFVertex(position: vector_float4(-widthScaling, heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(0.0, 0.0)),
            KFVertex(position: vector_float4(widthScaling, heightScaling, 0.0, 1.0), textureCoordinate: vector_float2(1.0, 0.0))
        ]
        
        vertices = device.makeBuffer(bytes: quadVertices, length: MemoryLayout<KFVertex>.stride * quadVertices.count, options: .storageModeShared)
        numVertices = quadVertices.count
    }
    
    /// 判断 YUV 数据是否为 full range
    private func pixelBufferIsFullRange(_ pixelBuffer: CVPixelBuffer) -> Bool {
        // 根据像素格式类型判断
        let pixelFormatType = CVPixelBufferGetPixelFormatType(pixelBuffer)
        
        // 检查是否是已知的全范围格式
        let isFullRange: Bool
        switch pixelFormatType {
        case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
            isFullRange = true
        case kCVPixelFormatType_420YpCbCr8PlanarFullRange:
            isFullRange = true
        default:
            isFullRange = false
        }
        
        return isFullRange
    }
    
    /// 渲染数据
    private func drawInMTKView(_ view: MTKView) {
        semaphore.wait()
        defer { semaphore.signal() }
        
        guard let pixelBuffer = self.pixelBuffer else { 
            return 
        }
        self.pixelBuffer = nil
        
        // 检查并初始化命令队列
        if commandQueue == nil {
            guard let device = mtkView.device else {
                return
            }
            commandQueue = device.makeCommandQueue()
        }
        
        guard let commandQueue = self.commandQueue,
              let renderPassDescriptor = view.currentRenderPassDescriptor,
              let drawable = view.currentDrawable else { 
            return 
        }
        
        let commandBuffer = commandQueue.makeCommandBuffer()
        
        guard let renderEncoder = commandBuffer?.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { 
            return 
        }
        
        renderEncoder.setViewport(MTLViewport(originX: 0.0, originY: 0.0,
                                             width: Double(viewportSize.x),
                                             height: Double(viewportSize.y),
                                             znear: -1.0, zfar: 1.0))
        
        let isRenderYUV = CVPixelBufferGetPlaneCount(pixelBuffer) > 1
        
        // 确保创建管道状态
        if pipelineState == nil {
            setupPipeline(isYUV: isRenderYUV)
        }
        
        guard let pipelineState = pipelineState else {
            renderEncoder.endEncoding()
            return
        }
        
        renderEncoder.setRenderPipelineState(pipelineState)
        
        
        
        
        
        if updateFillMode {
            updateVertices()
            updateFillMode = false
        }
        
        if let vertices = vertices {
            // 使用常量值0作为顶点缓冲索引，与Metal着色器中定义一致
            renderEncoder.setVertexBuffer(vertices, offset: 0, index: 0)
        } else {
            renderEncoder.endEncoding()
            return
        }
        
        if isRenderYUV {
            // 处理 YUV 纹理
            var textureY: MTLTexture?
            var textureUV: MTLTexture?
            
            if let textureCache = textureCache {
                // 确保pixelBuffer被锁定以便安全访问
                CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
                defer {
                    CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
                }
                
                let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, 0)
                let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, 0)
                let pixelFormat = MTLPixelFormat.r8Unorm
                
                var cvTextureY: CVMetalTexture?
                let statusY = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureY)
                
                if statusY == kCVReturnSuccess, let cvTextureY = cvTextureY {
                    textureY = CVMetalTextureGetTexture(cvTextureY)
                }
                
                let widthUV = CVPixelBufferGetWidthOfPlane(pixelBuffer, 1)
                let heightUV = CVPixelBufferGetHeightOfPlane(pixelBuffer, 1)
                let pixelFormatUV = MTLPixelFormat.rg8Unorm
                
                var cvTextureUV: CVMetalTexture?
                let statusUV = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormatUV, widthUV, heightUV, 1, &cvTextureUV)
                
                if statusUV == kCVReturnSuccess, let cvTextureUV = cvTextureUV {
                    textureUV = CVMetalTextureGetTexture(cvTextureUV)
                }
            }
            
            if let textureY = textureY, let textureUV = textureUV {
                // 设置纹理，使用shader中定义的索引
                renderEncoder.setFragmentTexture(textureY, index: 0)
                renderEncoder.setFragmentTexture(textureUV, index: 1)
            } else {
                renderEncoder.endEncoding()
                return
            }
            
            if yuvMatrix == nil {
                // 获取颜色空间信息，如果没有就使用默认值
                let colorSpace: CFTypeRef
                let isFullRange: Bool
                
                // 尝试从pixelBuffer获取YUV颜色空间信息
                if let matrixKey = CVBufferCopyAttachment(pixelBuffer, kCVImageBufferYCbCrMatrixKey, nil) {
                    colorSpace = matrixKey
                    isFullRange = pixelBufferIsFullRange(pixelBuffer)
                } else {
                    // 如果pixelBuffer中没有颜色空间信息，使用默认值
                    // 对于相机视频，通常使用BT.601标准和full range
                    colorSpace = kCVImageBufferYCbCrMatrix_ITU_R_601_4
                    
                    // 检查格式类型判断是否为full range
                    let format = CVPixelBufferGetPixelFormatType(pixelBuffer)
                    isFullRange = (format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
                }
                
                setupYUVMatrix(isFullRange: isFullRange, colorSpace: colorSpace)
            }
            
            // 设置矩阵缓冲区，使用shader中定义的索引
            if let yuvMatrix = yuvMatrix {
                renderEncoder.setFragmentBuffer(yuvMatrix, offset: 0, index: 0)
            }
            
        } else {
            // 处理 RGB 纹理
            var textureRGB: MTLTexture?
            
            if let textureCache = textureCache {
                let width = CVPixelBufferGetWidth(pixelBuffer)
                let height = CVPixelBufferGetHeight(pixelBuffer)
                let pixelFormat = MTLPixelFormat.bgra8Unorm
                
                var cvTextureRGB: CVMetalTexture?
                let status = CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault, textureCache, pixelBuffer, nil, pixelFormat, width, height, 0, &cvTextureRGB)
                
                if status == kCVReturnSuccess, let cvTextureRGB = cvTextureRGB {
                    textureRGB = CVMetalTextureGetTexture(cvTextureRGB)
                }
            }
            
            if let textureRGB = textureRGB {
                // 设置RGB纹理，使用shader中定义的索引
                renderEncoder.setFragmentTexture(textureRGB, index: 0)
            } else {
                renderEncoder.endEncoding()
                return
            }
        }
        
        renderEncoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: numVertices)
        renderEncoder.endEncoding()
        
        commandBuffer?.present(drawable)
        commandBuffer?.commit()
    }
}

// MARK: - MTKViewDelegate
extension KFMetalView: MTKViewDelegate {
    func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
        viewportSize = vector_uint2(UInt32(size.width), UInt32(size.height))
    }
    
    func draw(in view: MTKView) {
        // Metal 视图回调，有数据情况下渲染视图
        weak var weakSelf = self
        renderQueue.async {
            guard let self = weakSelf else { return }
            self.drawInMTKView(view)
        }
    }
}
相关推荐
97650333531 分钟前
iOS 审核 cocos 4.3a【苹果机审的“分层阈值”设计】
flutter·游戏·unity·ios
I烟雨云渊T31 分钟前
iOS Alamofire库的使用
ios
追随远方32 分钟前
Android OpenSL ES 音频播放完整实现指南
android·elasticsearch·音视频
程序员老刘·33 分钟前
iOS 26 beta1 真机无法执行hot reload
flutter·ios·跨平台开发·客户端开发
EndingCoder34 分钟前
React Native 构建与打包发布（iOS + Android）
android·react native·ios
Digitally43 分钟前
如何将视频从 iPhone 传输到 HP 笔记本电脑
电脑·音视频·iphone
程序员小刘1 小时前
HarmonyOS 5鸿蒙多端编译实战：从Android/iOS到HarmonyOS 5 的跨端迁移指南详
android·ios·华为·harmonyos
I烟雨云渊T1 小时前
iOS swiftUI的实用举例
ios·swiftui·swift
getapi3 小时前
将 App 安装到 iPhone 真机上测试
ios·iphone
键盘歌唱家3 小时前
AIGC方案-java实现视频伪动效果
java·aigc·音视频
热门推荐
01Coze扣子平台完整体验和实践（附国内和国际版对比）02【图像处理与机器视觉】XJTU期末考点03KGG转MP3工具|非KGM文件|解密音频04零代码入门 | Coze——让大模型接入自己的数据库05海康Visionmaster-常见问题排查方法-启动阶段06YOLOv8入门 | 重要性能衡量指标、训练结果评价及分析及影响mAP的因素【发论文关注的指标】07DeepSeek各版本说明与优缺点分析08从零安装 LLaMA-Factory 微调 Qwen 大模型成功及所有的坑09【SpeedAI科研小助手】2分钟极速解决知网维普重复率、AIGC率过高，一键全文降！文件格式不变，公式都保留的！10扣子（coze）实战|我用扣子搭建了一个自动分析小红薯笔记内容的AI应用|详细步骤拆解