UE5.4.2中使用NCNN库实时pose捕捉视频的方法示例

我救我自己2024-08-19 13:23

本文中使用的是ue5.4.2中官方自带的Opencv4.55，官方ncnn-20240410-windows-vs2022-shared库，ncnn-assets-master的模型库，进行实时pose捕捉，并将关键点显示在UI中。

需要创建ue c++ 项目，新建空白插件NCNNPlugin，复制bin,include,lib到插件目录下的Source\NCNNPlugin中,在NCNNPlugin.Build.cs中加载include、lib、dll,将dll复制到插件目录下的Plugins\NCNNPlugin\Binaries\Win64中（Plugins就是插件目录），再复制一份到项目目录下的XXX\Binaries\Win64中（XXX就是项目目录），其他都不用管，在项目的build.cs中开启插件NCNNPlugin，到这里就完成了插件的创建，NCNN库的加载。

NCNNPlugin.Build.cs

cs 复制代码 
// Copyright Epic Games, Inc. All Rights Reserved.

using System;
using UnrealBuildTool;
using System.IO;

public class NCNNPlugin : ModuleRules
{
	public NCNNPlugin(ReadOnlyTargetRules Target) : base(Target)
	{
		PCHUsage = ModuleRules.PCHUsageMode.UseExplicitOrSharedPCHs;
		string NCNNPath = Path.Combine(ModuleDirectory, "");
		
		Console.WriteLine("ModuleDirectory::::::" + ModuleDirectory);
		
		PublicIncludePaths.AddRange(
			new string[] {
				Path.Combine(ModuleDirectory,"include")
			}
			);
		
		PublicAdditionalLibraries.AddRange(new string[]
		{
			Path.Combine(ModuleDirectory,"lib","ncnn.lib"),
		});
		
		if (Target.Platform == UnrealTargetPlatform.Win64)
		{
			string DLLPath = Path.Combine(ModuleDirectory, "bin", "ncnn.dll");
			RuntimeDependencies.Add(DLLPath);
			Console.WriteLine("DLLPath::::::" + DLLPath);
			
			PublicDelayLoadDLLs.Add("ncnn.dll");
			
			// 将 DLL 文件复制到项目的二进制目录
			string ProjectBinariesDir = Path.Combine(Target.RelativeEnginePath, "Binaries", Target.Platform.ToString());
			RuntimeDependencies.Add(Path.Combine(ProjectBinariesDir, "ncnn.dll"), StagedFileType.NonUFS);

            PublicDefinitions.Add("__ARM_NEON=0");
            PublicDefinitions.Add("__SSE2__=0");
            PublicDefinitions.Add("__ANDROID_API__=0");
            PublicDefinitions.Add("__mips_msa=0");
            PublicDefinitions.Add("__loongarch_sx=0");
            PublicDefinitions.Add("__riscv_vector=0");
            
		}

        PrivateIncludePaths.AddRange(
			new string[] {
				// ... add other private include paths required here ...
			}
			);
			
		
		PublicDependencyModuleNames.AddRange(
			new string[]
			{
				"Core",
				// ... add other public dependencies that you statically link with here ...
			}
			);
			
		
		PrivateDependencyModuleNames.AddRange(
			new string[]
			{
				"CoreUObject",
				"Engine",
				"Slate",
				"SlateCore",
				// ... add private dependencies that you statically link with here ...	
			}
			);
		
		
		DynamicallyLoadedModuleNames.AddRange(
			new string[]
			{
				// ... add any modules that your module loads dynamically here ...
			}
			);
	}
}

项目的build.cs

cs 复制代码 
// Copyright Epic Games, Inc. All Rights Reserved.

using UnrealBuildTool;

public class NCNN_Thiredparty : ModuleRules
{
	public NCNN_Thiredparty(ReadOnlyTargetRules Target) : base(Target)
	{
		PCHUsage = PCHUsageMode.UseExplicitOrSharedPCHs;
	
		PublicDependencyModuleNames.AddRange(new string[]
		{
			"Core", 
			"CoreUObject", 
			"Engine", 
			"InputCore", 
			"EnhancedInput",
			"OpenCV",
			"OpenCVHelper",
			"UMG",
			"Slate",
			"SlateCore",
			"ImageWrapper",
			"NCNNPlugin"
	});

		PrivateDependencyModuleNames.AddRange(new string[] {  });

		// Uncomment if you are using Slate UI
		// PrivateDependencyModuleNames.AddRange(new string[] { "Slate", "SlateCore" });
		
		// Uncomment if you are using online features
		// PrivateDependencyModuleNames.Add("OnlineSubsystem");

		// To include OnlineSubsystemSteam, add it to the plugins section in your uproject file with the Enabled attribute set to true
	}
}

NCNN库的使用

创建Object类NcnnDataClass类，ActorComponent类NCNNTrackComponent，UserWidget类MyCPPWidget类（需要编辑器内创建），BP_UserWidget类（内添加image小部件CVImage）

NcnnDataClass.h(NcnnDataClass.cpp中不用修改)

cpp 复制代码 
// Fill out your copyright notice in the Description page of Project Settings.

#pragma once

#include "CoreMinimal.h"
#include "UObject/Object.h"

#if WITH_OPENCV
#include "PreOpenCVHeaders.h"
#include "opencv2/opencv.hpp"
#include "PostOpenCVHeaders.h"
#endif

#include "NcnnDataClass.generated.h"

struct KeyPoint {
	cv::Point2f p;
	float prob;
};

UCLASS()
class NCNN_THIREDPARTY_API UNcnnDataClass : public UObject
{
	GENERATED_BODY()
};

MyCPPWidget.h(MyCPPWidget.cpp中不用修改)

cpp 复制代码 
// Fill out your copyright notice in the Description page of Project Settings.

#pragma once

#include "CoreMinimal.h"
#include "Blueprint/UserWidget.h"
#include "Components/Image.h"
#include "MyCPPWidget.generated.h"

/**
 * 
 */
UCLASS()
class NCNN_THIREDPARTY_API UMyCPPWidget : public UUserWidget
{
	GENERATED_BODY()
public:
	virtual void NativeConstruct() override;

    //绑定BP_userwidget中的小部件
	UPROPERTY(BlueprintReadWrite,meta = (BindWidget))
	UImage* CVImage;
};

NCNNTrackComponent.h

cpp 复制代码 
// Fill out your copyright notice in the Description page of Project Settings.

#pragma once
#include "CoreMinimal.h"
#include "Components/ActorComponent.h"
#include "ncnn/net.h"

#if WITH_OPENCV
#include "PreOpenCVHeaders.h"
#include "opencv2/opencv.hpp"
#include "PostOpenCVHeaders.h"
#endif

#include "NcnnDataClass.h"

#include "NCNNTrackComponent.generated.h"

UCLASS(ClassGroup=(Custom), meta=(BlueprintSpawnableComponent))
class NCNN_THIREDPARTY_API UNCNNTrackComponent : public UActorComponent
{
	GENERATED_BODY()

public:
	// Sets default values for this component's properties
	UNCNNTrackComponent();

protected:
	// Called when the game starts
	virtual void BeginPlay() override;
	virtual void EndPlay(const EEndPlayReason::Type EndPlayReason) override;
	
public:
	// Called every frame
	virtual void TickComponent(float DeltaTime, ELevelTick TickType,
	                           FActorComponentTickFunction* ThisTickFunction) override;
	
	//NCNN
	ncnn::Net posenet;
	static int detect_posenet(const cv::Mat& bgr, std::vector<KeyPoint>& keypoints, ncnn::Net& posenet);
	static void draw_pose(cv::Mat& image, const std::vector<KeyPoint>& keypoints, int radius = 3, const cv::Scalar& color = cv::Scalar(0, 255, 0));
	
	//UI
    //这里需要在创建一个ACtor并添加NCNNTrack组件后在编辑器内指认BP_widget,目的是将该widget->AddToViewport();

	UPROPERTY(EditAnywhere,BlueprintReadWrite)
	TSubclassOf<UUserWidget> BP_widget;

	UPROPERTY(BlueprintReadOnly)
	UTexture2D* CVLoadImage;

	// OpenCV VideoCapture
	cv::VideoCapture VideoCapture;
	float FrameTime;
	bool bIsVideoPlaying = false;
	float AccumulatedTime = 0.0f;
	cv::Mat Frame;
	cv::Mat FrameBGRA;
};

NCNNTrackComponent.cpp

cpp 复制代码 
#include "NCNNTrackComponent.h"

#include "MyCPPWidget.h"
// 取消冲突宏定义
#undef UpdateResource

// Sets default values for this component's properties
UNCNNTrackComponent::UNCNNTrackComponent()
{
    PrimaryComponentTick.bCanEverTick = true;
    posenet.opt.use_vulkan_compute = true;
    posenet.load_param("D:/ue_temp/NCNN_Thiredparty/pose.param");
    posenet.load_model("D:/ue_temp/NCNN_Thiredparty/pose.bin");
}

// Called when the game starts
void UNCNNTrackComponent::BeginPlay()
{
    Super::BeginPlay();
    
    //加载视频
    FString filepath = TEXT("d:/dance.mp4");
    UE_LOG(LogTemp, Warning, TEXT("尝试加载视频文件: %s"), *filepath);

    // 使用 OpenCV 读取视频
    VideoCapture.open(TCHAR_TO_UTF8(*filepath));
    if (!VideoCapture.isOpened())
    {
        UE_LOG(LogTemp, Warning, TEXT("无法加载视频文件: %s"), *filepath);
        return;
    }
    UE_LOG(LogTemp, Warning, TEXT("成功加载视频文件: %s"), *filepath);

    // 从视频文件中获取帧速率
    double FPS = VideoCapture.get(cv::CAP_PROP_FPS);
    if (FPS <= 0)
    {
        UE_LOG(LogTemp, Warning, TEXT("无法获取帧速率，默认设置为30 FPS"));
        FPS = 30.0; // 如果获取失败，默认设置为30 FPS
    }
    FrameTime = 1.0f / FPS;
    bIsVideoPlaying = true;
    UE_LOG(LogTemp, Warning, TEXT("视频帧速率: %f FPS"), FPS);

    // 初始化纹理
    int width = static_cast<int>(VideoCapture.get(cv::CAP_PROP_FRAME_WIDTH));
    int height = static_cast<int>(VideoCapture.get(cv::CAP_PROP_FRAME_HEIGHT));

    UTexture2D* Texture2D = UTexture2D::CreateTransient(width, height, PF_B8G8R8A8);
    if (!Texture2D)
    {
        UE_LOG(LogTemp, Warning, TEXT("纹理创建失败"));
        return;
    }

    CVLoadImage = Texture2D;

    // 创建并设置 UI 小部件
    UUserWidget* bpwidget = CreateWidget<UUserWidget>(GetWorld(), BP_widget);
    UMyCPPWidget* CPPWidget = Cast<UMyCPPWidget>(bpwidget);
    if (CPPWidget && CPPWidget->CVImage)
    {
        CPPWidget->CVImage->SetBrushFromTexture(CVLoadImage);
        CPPWidget->CVImage->Brush.ImageSize = FVector2D(CVLoadImage->GetSizeX(), CVLoadImage->GetSizeY());
    }

    bpwidget->AddToViewport();
}

void UNCNNTrackComponent::EndPlay(const EEndPlayReason::Type EndPlayReason)
{
    Super::EndPlay(EndPlayReason);
    if (VideoCapture.isOpened())
    {
        VideoCapture.release();
        UE_LOG(LogTemp, Warning, TEXT("VideoCapture released successfully"));
    }

    bIsVideoPlaying = false;
}

void UNCNNTrackComponent::TickComponent(float DeltaTime, ELevelTick TickType,
                                        FActorComponentTickFunction* ThisTickFunction)
{
    Super::TickComponent(DeltaTime, TickType, ThisTickFunction);
    
    if (!VideoCapture.isOpened() || !bIsVideoPlaying)
    {
        return;
    }
 
    // 累积时间
    AccumulatedTime += DeltaTime;
 
    if (AccumulatedTime >= FrameTime)
    {
        if (VideoCapture.read(Frame))
        {
            // 调整帧大小至 256x512,这里需要注意视频尺寸需要与模型要求尺寸保持一致关键点才能在正确位置
            cv::resize(Frame, Frame, cv::Size(256, 512));
            std::vector<KeyPoint> keypoints;
            detect_posenet(Frame, keypoints, posenet);

            // 如果 FrameBGRA 为空或尺寸不匹配，则初始化
            if (FrameBGRA.empty() || FrameBGRA.size() != Frame.size())
            {
                FrameBGRA = cv::Mat(Frame.size(), CV_8UC4);
            }

            // 转换 BGR 图像为 BGRA 格式，并在 FrameBGRA 中保存
            cv::cvtColor(Frame, FrameBGRA, cv::COLOR_BGR2BGRA);
            draw_pose(FrameBGRA, keypoints);

            // 更新纹理
            FTexture2DMipMap& MipMap = CVLoadImage->GetPlatformData()->Mips[0];
            void* TextureData = MipMap.BulkData.Lock(LOCK_READ_WRITE);
            int32 SrcPitch = FrameBGRA.cols * FrameBGRA.elemSize();
            int32 DstPitch = MipMap.SizeX * 4;

            for (int32 Row = 0; Row < FrameBGRA.rows; ++Row)
            {
                FMemory::Memcpy(static_cast<uint8*>(TextureData) + Row * DstPitch, FrameBGRA.ptr(Row), SrcPitch);
            }

            MipMap.BulkData.Unlock();
            CVLoadImage->UpdateResource();
        }

        // 重置累积时间
        AccumulatedTime -= FrameTime;
    }
 }

int UNCNNTrackComponent::detect_posenet(const cv::Mat& bgr, std::vector<KeyPoint>& keypoints, ncnn::Net& posenet)
{
    int w = bgr.cols;
    int h = bgr.rows;

    // 使用原始尺寸创建输入张量
    ncnn::Mat in = ncnn::Mat::from_pixels(bgr.data, ncnn::Mat::PIXEL_BGR2RGB, w, h);

    // 数据预处理：减去均值和归一化
    const float mean_vals[3] = { 0.485f * 255.f, 0.456f * 255.f, 0.406f * 255.f };
    const float norm_vals[3] = { 1 / 0.229f / 255.f, 1 / 0.224f / 255.f, 1 / 0.225f / 255.f };
    in.substract_mean_normalize(mean_vals, norm_vals);

    ncnn::Extractor ex = posenet.create_extractor();
    ex.input("data", in);

    ncnn::Mat out;
    ex.extract("conv3_fwd", out);

    // 解析heatmap以获取关键点
    keypoints.clear();
    for (int p = 0; p < out.c; p++) {
        const ncnn::Mat m = out.channel(p);

        float max_prob = 0.f;
        int max_x = 0;
        int max_y = 0;
        for (int y = 0; y < m.h; y++) {
            const float* ptr = m.row(y);
            for (int x = 0; x < m.w; x++) {
                float prob = ptr[x];
                if (prob > max_prob) {
                    max_prob = prob;
                    max_x = x;
                    max_y = y;
                }
            }
        }

        KeyPoint keypoint;
        keypoint.p = cv::Point2f(max_x * w / (float)m.w, max_y * h / (float)m.h);  // 按照原始图像尺寸调整关键点位置
        keypoint.prob = max_prob;

        keypoints.push_back(keypoint);
    }

    return 0;
}

void UNCNNTrackComponent::draw_pose(cv::Mat& image, const std::vector<KeyPoint>& keypoints, int radius,
    const cv::Scalar& color)
{
    // draw joint
    for (size_t i = 0; i < keypoints.size(); i++) {
        const KeyPoint& keypoint = keypoints[i];

        fprintf(stderr, "%.2f %.2f = %.5f\n", keypoint.p.x, keypoint.p.y, keypoint.prob);

        if (keypoint.prob < 0.5f)
            continue;

        // 使用自定义半径和颜色绘制关键点
        cv::circle(image, keypoint.p, radius, color, -1);
    }
}
相关推荐
陈友松2 小时前
UE5 文件传输插件：支持任意类型，任意大小(超过2G)的文件上传和下载
ue5
AI视觉网奇16 小时前
CBAudio2Face
ue5·audio2face
AI视觉网奇17 小时前
ue 操作 metahuman
ue5
AI视觉网奇20 小时前
ue python脚本 获取资产
笔记·ue5
AI视觉网奇21 小时前
audio2face docker方式
docker·ue5
会思考的猴子1 天前
UE5 笔记二 GameplayAbilitySystem Dash(冲刺)
笔记·ue5
AI视觉网奇2 天前
audio2face ue插件形式实战笔记
笔记·ue5
nutriu3 天前
从UE5.6DNA 导出指定LOD层级的ARkit52个表情或者Metahuman263个表情教程 #BlendShapeExporter
ue5·数字人·arkit·blendshape·虚拟角色·meta human·dna
AI视觉网奇3 天前
nvcr.io 登录方法
docker·ue5
会思考的猴子3 天前
UE5 C++ 笔记 GameplayAbilitySystem人物角色
c++·笔记·ue5
热门推荐
01GitHub 镜像站点02从快手“12·22”直播攻击事件看：一次教科书式的业务层饱和攻击03电脑检测软件—图吧工具箱043D 圣诞树网页代码05Linux下V2Ray安装配置指南06Web安全中SQL注入绕过WAF的具体手法和实战案例07UV安装并设置国内源08在VSCode配置Java开发环境的保姆级教程（适配各类AI编程IDE）09jdk21下载、安装（Windows、Linux、macOS）10SQLmap 完整使用指南：环境搭建 + 命令详解 + 实操案例