EmguCV学习笔记 VB.Net 11.9 姿势识别 OpenPose

版权声明:本文为博主原创文章,转载请在显著位置标明本文出处以及作者网名,未经作者允许不得用于商业目的。

EmguCV是一个基于OpenCV的开源免费的跨平台计算机视觉库,它向C#和VB.NET开发者提供了OpenCV库的大部分功能。

教程VB.net版本请访问: EmguCV学习笔记 VB.Net 目录-CSDN博客

教程C#版本请访问: EmguCV学习笔记 C# 目录-CSDN博客

笔者的博客网址:https://blog.csdn.net/uruseibest

教程配套文件及相关说明以及如何获得pdf教程和代码,请移步: EmguCV学习笔记

学习VB.Net知识,请移步: vb.net 教程 目录_vb中如何用datagridview-CSDN博客

学习C#知识,请移步: C# 教程 目录_c#教程目录-CSDN博客

11.9 姿势识别 OpenPose

OpenPose模型是一种用于人体姿态估计的深度学习模型,它能够检测出包括眼睛、鼻子、手臂、腿等18个人体的关键点,并估计它们的坐标位置和置信度。

0:Nose(鼻子)、1:neck(脖子)、2:rshoulder(右肩)、3:relbow(右肘部)、4:rwrist(右手腕)、5:shoulder(左肩)、6:lelbow(左肘部)、7:lwrist(左手腕)、8:rhip(右臀部)、9:rknee(右膝盖)、10:rankle(右脚踝)、11:lhip(左臀部)、12:lknee(左膝盖)、13:lankle(左脚踝)、14:reye(右眼)、15:leye(左眼)、16:rear(右耳)、17:lear(左耳)、18:background(背景,主要是作为下一步扩展使用,在实际中不处理)

在使用OpenPose模型时,通常需要将输入图像作为模型的输入,经过处理后得到一个四维数组作为输出结果。这个四维数组,其维度为(N, P, H, W),

各个维度的含义:

  1. N:在输入图像中检测到的人体数量。
  2. P:估计的关键点数,包括了人体的身体部位和手指关节等,只需要取前18个。
  3. H:关键点的坐标信息在输出结果中的高度,在实际使用中就是DnnInvoke.BlobFromImage中size参数设置输出的Height,如果最终输出到源图像,那么应该按照比例进行还原。
  4. W:关键点的坐标信息在输出结果中的宽度,在实际使用中就是DnnInvoke.BlobFromImage中size参数设置输出的Width,如果最终输出到源图像,那么应该按照比例进行还原。

具体到某个元素的值就是该点是人体关键点的置信度,例如(0,2,10,30)返回(0,2,height,width)中的最大值为0.759,那么可以认为坐标(10,30)是右肩的可能性为75.9%。

从上面可以看出,在这个四维数组中,每个元素包含了x坐标(维度W)、y坐标(维度H)和置信度三个值。因此,可以通过遍历该四维数组并解析每个元素来获取所有关键点的坐标信息和置信度,从而进行人体姿态估计的后续处理。

【代码位置:frmChapter11】Button10_Click、getMaxPoint

'关键点信息

Structure Keypoint

Dim conf As Single '置信度

Dim p As Point '关键点坐标

End Structure

'使用openpose显示人体关键点

Private Sub Button10_Click(sender As Object, e As EventArgs) Handles Button10.Click

'人体关键点

Dim body_Keypoint() As String = {"nose", "neck", "rshoulder", "relbow", "rwrist", "lshoulder",

"lelbow", "lwrist", "rhip", "rknee", "rankle", "lhip",

"lknee", "lankle", "reye", "leye", "rear", "lear", "background"}

Dim m As New Mat("C:\learnEmgucv\action.jpg", ImreadModes.Color)

Dim net As Dnn.Net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\openpose\graph_opt.pb")

Dim blob As Mat = DnnInvoke.BlobFromImage(m, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), False, False)

net.SetInput(blob)

Dim mout As Mat = net.Forward()

'返回四维数组

Dim fout(,,,) As Single

fout = mout.GetData()

Dim H As Integer = fout.GetLength(2)

Dim W As Integer = fout.GetLength(3)

Dim lkeypoint As New List(Of Keypoint)

'获得关键点信息

lkeypoint = getMaxPoint(fout)

Dim x, y As Single

For i As Integer = 0 To lkeypoint.Count - 1

'按照比例获得关键点在源图像中的坐标

x = (lkeypoint(i).p.X / W) * m.Width

y = (lkeypoint(i).p.Y / H) * m.Height

'调试时输出信息

'Console.WriteLine(body_Keypoint(i) & " " & lkeypoint(i).conf & " " & lkeypoint(i).p.X & "-" & lkeypoint(i).p.Y)

'置信度超过某个值才能认为是正确的结果

If lkeypoint(i).conf > 0.1 Then

CvInvoke.Circle(m, New Point(x, y), 4, New MCvScalar(255, 0, 0), -1)

End If

Next

CvInvoke.Imshow("m", m)

End Sub

'获得人体18个关键点列表,这里考虑只有一个人体的情况

Private Function getMaxPoint(ByVal inputarray(,,,) As Single) As List(Of Keypoint)

Dim lkeypoint As New List(Of Keypoint)

Dim peoplecount As Integer = 1 '考虑只有一个人体的情况,如果多个人体,请使用 inputarray.GetLength(0)

Dim modecount As Integer = 18 '只考虑18个人体关键点, inputarray.GetLength(1)

Dim dim3 As Integer = inputarray.GetLength(2) '图像高度

Dim dim4 As Integer = inputarray.GetLength(3) '图像宽度

'循环,检测到的人体个数

For i As Integer = 0 To peoplecount - 1

'循环,检测到的人体关键点

For j As Integer = 0 To modecount - 1

Dim maxvalue As Single = 0

Dim maxX As Integer = 0

Dim maxY As Integer = 0

'循环,图像高度,即对应Y坐标

For k As Integer = 0 To dim3 - 1

'循环,图像宽度,即对应X坐标

For l As Integer = 0 To dim4 - 1

'获得置信度最大的值,并获得其坐标

If maxvalue < inputarray(i, j, k, l) Then

maxvalue = inputarray(i, j, k, l)

maxX = l

maxY = k

End If

Next

Next

'添加到关键点列表

Dim kp As New Keypoint

kp.conf = maxvalue

kp.p = New Point(maxX, maxY)

lkeypoint.Add(kp)

Next

Next

Return lkeypoint

End Function

输出结果如下图所示:

图11-9 获得人体关键点

下面的代码通过人体关键点的关联,建立关键点的连线。

【代码位置:frmChapter11】Button11_Click、PointFToPoint

'关键点关联

Structure Relation

'开始关键点

Dim startpoint As Integer

'结束关键点

Dim endpoint As Integer

Sub New(ByVal startpoint As Integer, ByVal endpoint As Integer)

Me.startpoint = startpoint

Me.endpoint = endpoint

End Sub

End Structure

'获得人体关键点,并将关键点关联起来

Private Sub Button11_Click(sender As Object, e As EventArgs) Handles Button11.Click

Dim body_Keypoint() As String = {"nose", "neck", "rshoulder", "relbow", "rwrist", "lshoulder",

"lelbow", "lwrist", "rhip", "rknee", "rankle", "lhip",

"lknee", "lankle", "reye", "leye", "rear", "lear", "background"}

'18个关键点关联

Dim body_Relations As New List(Of Relation)

body_Relations.Add(New Relation(16, 14))

body_Relations.Add(New Relation(14, 0))

body_Relations.Add(New Relation(17, 15))

body_Relations.Add(New Relation(15, 0))

body_Relations.Add(New Relation(0, 1))

body_Relations.Add(New Relation(1, 2))

body_Relations.Add(New Relation(2, 3))

body_Relations.Add(New Relation(3, 4))

body_Relations.Add(New Relation(1, 5))

body_Relations.Add(New Relation(5, 6))

body_Relations.Add(New Relation(6, 7))

body_Relations.Add(New Relation(1, 8))

body_Relations.Add(New Relation(8, 9))

body_Relations.Add(New Relation(9, 10))

body_Relations.Add(New Relation(1, 11))

body_Relations.Add(New Relation(11, 12))

body_Relations.Add(New Relation(12, 13))

Dim m As New Mat("C:\learnEmgucv\action.jpg", ImreadModes.Color)

Dim net As Dnn.Net

net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\openpose\graph_opt.pb")

Dim blob As Mat

blob = DnnInvoke.BlobFromImage(m, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), False, False)

net.SetInput(blob)

Dim mout As New Mat

mout = net.Forward()

Dim fout(,,,) As Single

fout = mout.GetData()

Dim H As Integer = fout.GetLength(2)

Dim W As Integer = fout.GetLength(3)

Dim lkeypoint As New List(Of Keypoint)

lkeypoint = getMaxPoint(fout)

Dim x, y As Single

For i As Integer = 0 To lkeypoint.Count - 1

'按照比例获得关键点在源图像中的坐标

x = (lkeypoint(i).p.X / W) * m.Width

y = (lkeypoint(i).p.Y / H) * m.Height

'置信度超过某个值才能认为是正确的结果

If lkeypoint(i).conf > 0.1 Then

CvInvoke.Circle(m, New Point(x, y), 5, New MCvScalar(255, 0, 0), -1)

End If

Next

Dim startpoint As PointF

Dim startpointx, startpointy As Single

Dim endpoint As PointF

Dim endpointx, endpointy As Single

For Each body_Relation As Relation In body_Relations

startpointx = (lkeypoint(body_Relation.startpoint).p.X / W) * m.Width

startpointy = (lkeypoint(body_Relation.startpoint).p.Y / H) * m.Height

startpoint = New PointF(startpointx, startpointy)

endpointx = (lkeypoint(body_Relation.endpoint).p.X / W) * m.Width

endpointy = (lkeypoint(body_Relation.endpoint).p.Y / H) * m.Height

endpoint = New PointF(endpointx, endpointy)

'关键点置信度是否符合要求

If lkeypoint(body_Relation.startpoint).conf > 0.1 And lkeypoint(body_Relation.endpoint).conf > 0.1 Then

'关键点建立连线

CvInvoke.Line(m, PointFToPoint(startpoint), PointFToPoint(endpoint), New MCvScalar(0, 255, 0), 3)

End If

Next

CvInvoke.Imshow("m", m)

End Sub

'将PointF转Point方法

Public Shared Function PointFToPoint(ByVal pf As PointF) As Point

Return New Point(CInt(pf.X), CInt(pf.Y))

End Function

输出结果如下图所示:

图11-10 人体关键点连线

下面代码是在上面代码基础上,实现在视频中显示人体关键点的连线。

【代码位置:frmChapter11】Button12_Click、vc_ImageGrabbed

Dim vc As VideoCapture

Dim body_Relations As New List(Of Relation)

'将视频人物标注人体关键点和关键点连线

Private Sub Button12_Click(sender As Object, e As EventArgs) Handles Button12.Click

'18个关键点关联

body_Relations = New List(Of Relation)

body_Relations.Add(New Relation(16, 14))

body_Relations.Add(New Relation(14, 0))

body_Relations.Add(New Relation(17, 15))

body_Relations.Add(New Relation(15, 0))

body_Relations.Add(New Relation(0, 1))

body_Relations.Add(New Relation(1, 2))

body_Relations.Add(New Relation(2, 3))

body_Relations.Add(New Relation(3, 4))

body_Relations.Add(New Relation(1, 5))

body_Relations.Add(New Relation(5, 6))

body_Relations.Add(New Relation(6, 7))

body_Relations.Add(New Relation(1, 8))

body_Relations.Add(New Relation(8, 9))

body_Relations.Add(New Relation(9, 10))

body_Relations.Add(New Relation(1, 11))

body_Relations.Add(New Relation(11, 12))

body_Relations.Add(New Relation(12, 13))

vc = New VideoCapture("C:\learnEmgucv\action.mp4")

If vc.IsOpened = False Then

MessageBox.Show("打开文件失败")

Exit Sub

End If

'添加ImageGrabbed事件

AddHandler vc.ImageGrabbed, AddressOf vc_ImageGrabbed

vc.Start()

End Sub

Private Sub vc_ImageGrabbed(sender As Object, e As EventArgs)

Dim outangle As Double = 0

Dim outpix As Double = 0

Dim nextframe As New Mat

vc.Retrieve(nextframe)

If vc.Get(CapProp.PosFrames) >= vc.Get(CapProp.FrameCount) Then

vc.Stop()

vc.Dispose()

RemoveHandler vc.ImageGrabbed, AddressOf vc_ImageGrabbed

Exit Sub

End If

Dim net As Dnn.Net

net = DnnInvoke.ReadNetFromTensorflow("graph_opt.pb")

Dim blob As Mat

blob = DnnInvoke.BlobFromImage(nextframe, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), True, False)

net.SetInput(blob)

Dim mout As New Mat

mout = net.Forward()

Dim fout(,,,) As Single

fout = mout.GetData()

Dim H As Integer = fout.GetLength(2)

Dim W As Integer = fout.GetLength(3)

Dim lkeypoint As New List(Of Keypoint)

lkeypoint = getMaxPoint(fout)

Dim x, y As Single

For i As Integer = 0 To lkeypoint.Count - 1

x = (lkeypoint(i).p.X / W) * nextframe.Width

y = (lkeypoint(i).p.Y / H) * nextframe.Height

If lkeypoint(i).conf > 0.1 Then

CvInvoke.Circle(nextframe, New Point(x, y), 5, New MCvScalar(0, 0, 255), -1)

End If

Next

Dim startpoint As PointF

Dim startpointx, startpointy As Single

Dim endpoint As PointF

Dim endpointx, endpointy As Single

For Each body_Relation As Relation In body_Relations

startpointx = (lkeypoint(body_Relation.startpoint).p.X / W) * nextframe.Width

startpointy = (lkeypoint(body_Relation.startpoint).p.Y / H) * nextframe.Height

startpoint = New PointF(startpointx, startpointy)

endpointx = (lkeypoint(body_Relation.endpoint).p.X / W) * nextframe.Width

endpointy = (lkeypoint(body_Relation.endpoint).p.Y / H) * nextframe.Height

endpoint = New PointF(endpointx, endpointy)

If lkeypoint(body_Relation.startpoint).conf > 0.15 And lkeypoint(body_Relation.endpoint).conf > 0.15 Then

CvInvoke.Line(nextframe, PointFToPoint(startpoint), PointFToPoint(endpoint), New MCvScalar(0, 255, 0), 4)

End If

Next

ImageBox1.Image = nextframe

Threading.Thread.Sleep(30)

End Sub

输出结果如下图所示:

图11-11 视频中使用人体关键点连线

相关推荐
BruceNeter1 天前
C# winform卡死问题分析1——线程池子线程创建句柄
c#·.net
何以解忧唯有撸码3 天前
开源】当桌面图标开始打台球:我用 WinForms 做了一桌“真的”台球
c#·winform·自定义控件·gdi
国家一级摸鱼选手3 天前
[Unity]几何碰撞检测:SAT 分离轴定理详解(从数学原理到 C# 实现)
unity·c#·游戏引擎·分离轴定理(sat)·多边形相交
汤姆宝库4 天前
BPE分词器的完整实现:字节对合并与词表构建
开发语言·人工智能·c#
Risehuxyc4 天前
C# 发展历史
开发语言·c#
kyle~4 天前
C# --- 反射
开发语言·c#
硅谷秋水4 天前
MorphMimic 方法分析
人工智能·机器学习·计算机视觉·机器人
jay神4 天前
基于 YOLOv8 的传送带异物检测系统(Web 版)
前端·人工智能·深度学习·yolo·计算机视觉·计算机毕业设计
暴走约伯5 天前
【VFX工具链源码拆解 09】终极工业化闭环:基于 ID 规则驱动的全自动场景建档、骨骼挂载与男转女场景迁移
unity·c#·游戏引擎·材质
用户788477316346 天前
WinForms 迁移 WPF:Upgrade Assistant 覆盖 30%,剩下 70% 我手写了六周
c#