EmguCV学习笔记 VB.Net 11.9 姿势识别 OpenPose

EmguCV是一个基于OpenCV的开源免费的跨平台计算机视觉库,它向C#和VB.NET开发者提供了OpenCV库的大部分功能。

教程VB.net版本请访问： EmguCV学习笔记 VB.Net 目录-CSDN博客

教程C#版本请访问： EmguCV学习笔记 C# 目录-CSDN博客

笔者的博客网址：https://blog.csdn.net/uruseibest

教程配套文件及相关说明以及如何获得pdf教程和代码，请移步： EmguCV学习笔记

学习VB.Net知识，请移步： vb.net 教程目录_vb中如何用datagridview-CSDN博客

学习C#知识，请移步： C# 教程目录_c#教程目录-CSDN博客

11.9 姿势识别 OpenPose

OpenPose模型是一种用于人体姿态估计的深度学习模型，它能够检测出包括眼睛、鼻子、手臂、腿等18个人体的关键点，并估计它们的坐标位置和置信度。

0：Nose（鼻子）、1：neck（脖子）、2：rshoulder（右肩）、3：relbow（右肘部）、4：rwrist（右手腕）、5：shoulder（左肩）、6：lelbow（左肘部）、7：lwrist（左手腕）、8：rhip（右臀部）、9：rknee（右膝盖）、10：rankle（右脚踝）、11：lhip（左臀部）、12：lknee（左膝盖）、13：lankle（左脚踝）、14：reye（右眼）、15：leye（左眼）、16：rear（右耳）、17：lear（左耳）、18：background（背景，主要是作为下一步扩展使用，在实际中不处理）

在使用OpenPose模型时，通常需要将输入图像作为模型的输入，经过处理后得到一个四维数组作为输出结果。这个四维数组，其维度为(N, P, H, W)，

各个维度的含义：

N：在输入图像中检测到的人体数量。
P：估计的关键点数，包括了人体的身体部位和手指关节等，只需要取前18个。
H：关键点的坐标信息在输出结果中的高度，在实际使用中就是DnnInvoke.BlobFromImage中size参数设置输出的Height，如果最终输出到源图像，那么应该按照比例进行还原。
W：关键点的坐标信息在输出结果中的宽度，在实际使用中就是DnnInvoke.BlobFromImage中size参数设置输出的Width，如果最终输出到源图像，那么应该按照比例进行还原。

具体到某个元素的值就是该点是人体关键点的置信度，例如（0,2,10,30）返回（0,2,height,width）中的最大值为0.759，那么可以认为坐标(10,30)是右肩的可能性为75.9%。

从上面可以看出，在这个四维数组中，每个元素包含了x坐标（维度W）、y坐标（维度H）和置信度三个值。因此，可以通过遍历该四维数组并解析每个元素来获取所有关键点的坐标信息和置信度，从而进行人体姿态估计的后续处理。

【代码位置：frmChapter11】Button10_Click、getMaxPoint

'关键点信息

Structure Keypoint

Dim conf As Single '置信度

Dim p As Point '关键点坐标

End Structure

'使用openpose显示人体关键点

Private Sub Button10_Click(sender As Object, e As EventArgs) Handles Button10.Click

'人体关键点

Dim body_Keypoint() As String = {"nose", "neck", "rshoulder", "relbow", "rwrist", "lshoulder",

"lelbow", "lwrist", "rhip", "rknee", "rankle", "lhip",

"lknee", "lankle", "reye", "leye", "rear", "lear", "background"}

Dim m As New Mat("C:\learnEmgucv\action.jpg", ImreadModes.Color)

Dim net As Dnn.Net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\openpose\graph_opt.pb")

Dim blob As Mat = DnnInvoke.BlobFromImage(m, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), False, False)

net.SetInput(blob)

Dim mout As Mat = net.Forward()

'返回四维数组

Dim fout(,,,) As Single

fout = mout.GetData()

Dim H As Integer = fout.GetLength(2)

Dim W As Integer = fout.GetLength(3)

Dim lkeypoint As New List(Of Keypoint)

'获得关键点信息

lkeypoint = getMaxPoint(fout)

Dim x, y As Single

For i As Integer = 0 To lkeypoint.Count - 1

'按照比例获得关键点在源图像中的坐标

x = (lkeypoint(i).p.X / W) * m.Width

y = (lkeypoint(i).p.Y / H) * m.Height

'调试时输出信息

'Console.WriteLine(body_Keypoint(i) & " " & lkeypoint(i).conf & " " & lkeypoint（i).p.X & "-" & lkeypoint(i).p.Y)

'置信度超过某个值才能认为是正确的结果

If lkeypoint(i).conf > 0.1 Then

CvInvoke.Circle(m, New Point(x, y), 4, New MCvScalar(255, 0, 0), -1)

End If

CvInvoke.Imshow("m", m)

End Sub

'获得人体18个关键点列表，这里考虑只有一个人体的情况

Private Function getMaxPoint(ByVal inputarray(,,,) As Single) As List(Of Keypoint)

Dim lkeypoint As New List(Of Keypoint)

Dim peoplecount As Integer = 1 '考虑只有一个人体的情况，如果多个人体，请使用 inputarray.GetLength(0)

Dim modecount As Integer = 18 '只考虑18个人体关键点， inputarray.GetLength(1)

Dim dim3 As Integer = inputarray.GetLength(2) '图像高度

Dim dim4 As Integer = inputarray.GetLength(3) '图像宽度

'循环，检测到的人体个数

For i As Integer = 0 To peoplecount - 1

'循环，检测到的人体关键点

For j As Integer = 0 To modecount - 1

Dim maxvalue As Single = 0

Dim maxX As Integer = 0

Dim maxY As Integer = 0

'循环，图像高度，即对应Y坐标

For k As Integer = 0 To dim3 - 1

'循环，图像宽度，即对应X坐标

For l As Integer = 0 To dim4 - 1

'获得置信度最大的值，并获得其坐标

If maxvalue < inputarray(i, j, k, l) Then

maxvalue = inputarray(i, j, k, l)

maxX = l

maxY = k

End If

'添加到关键点列表

Dim kp As New Keypoint

kp.conf = maxvalue

kp.p = New Point(maxX, maxY)

lkeypoint.Add(kp)

Return lkeypoint

End Function

输出结果如下图所示：

图11-9 获得人体关键点

下面的代码通过人体关键点的关联，建立关键点的连线。

【代码位置：frmChapter11】Button11_Click、PointFToPoint

'关键点关联

Structure Relation

'开始关键点

Dim startpoint As Integer

'结束关键点

Dim endpoint As Integer

Sub New(ByVal startpoint As Integer, ByVal endpoint As Integer)

Me.startpoint = startpoint

Me.endpoint = endpoint

End Sub

End Structure

'获得人体关键点，并将关键点关联起来

Private Sub Button11_Click(sender As Object, e As EventArgs) Handles Button11.Click

Dim body_Keypoint() As String = {"nose", "neck", "rshoulder", "relbow", "rwrist", "lshoulder",

"lelbow", "lwrist", "rhip", "rknee", "rankle", "lhip",

"lknee", "lankle", "reye", "leye", "rear", "lear", "background"}

'18个关键点关联

Dim body_Relations As New List(Of Relation)

body_Relations.Add(New Relation(16, 14))

body_Relations.Add(New Relation(14, 0))

body_Relations.Add(New Relation(17, 15))

body_Relations.Add(New Relation(15, 0))

body_Relations.Add(New Relation(0, 1))

body_Relations.Add(New Relation(1, 2))

body_Relations.Add(New Relation(2, 3))

body_Relations.Add(New Relation(3, 4))

body_Relations.Add(New Relation(1, 5))

body_Relations.Add(New Relation(5, 6))

body_Relations.Add(New Relation(6, 7))

body_Relations.Add(New Relation(1, 8))

body_Relations.Add(New Relation(8, 9))

body_Relations.Add(New Relation(9, 10))

body_Relations.Add(New Relation(1, 11))

body_Relations.Add(New Relation(11, 12))

body_Relations.Add(New Relation(12, 13))

Dim m As New Mat("C:\learnEmgucv\action.jpg", ImreadModes.Color)

Dim net As Dnn.Net

net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\openpose\graph_opt.pb")

Dim blob As Mat

blob = DnnInvoke.BlobFromImage(m, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), False, False)

net.SetInput(blob)

Dim mout As New Mat

mout = net.Forward()

Dim fout(,,,) As Single

fout = mout.GetData()

Dim H As Integer = fout.GetLength(2)

Dim W As Integer = fout.GetLength(3)

Dim lkeypoint As New List(Of Keypoint)

lkeypoint = getMaxPoint(fout)

Dim x, y As Single

For i As Integer = 0 To lkeypoint.Count - 1

'按照比例获得关键点在源图像中的坐标

x = (lkeypoint(i).p.X / W) * m.Width

y = (lkeypoint(i).p.Y / H) * m.Height

'置信度超过某个值才能认为是正确的结果

If lkeypoint(i).conf > 0.1 Then

CvInvoke.Circle(m, New Point(x, y), 5, New MCvScalar(255, 0, 0), -1)

End If

Dim startpoint As PointF

Dim startpointx, startpointy As Single

Dim endpoint As PointF

Dim endpointx, endpointy As Single

For Each body_Relation As Relation In body_Relations

startpointx = (lkeypoint(body_Relation.startpoint).p.X / W) * m.Width

startpointy = (lkeypoint(body_Relation.startpoint).p.Y / H) * m.Height

startpoint = New PointF(startpointx, startpointy)

endpointx = (lkeypoint(body_Relation.endpoint).p.X / W) * m.Width

endpointy = (lkeypoint(body_Relation.endpoint).p.Y / H) * m.Height

endpoint = New PointF(endpointx, endpointy)

'关键点置信度是否符合要求

If lkeypoint(body_Relation.startpoint).conf > 0.1 And lkeypoint(body_Relation.endpoint).conf > 0.1 Then

'关键点建立连线

CvInvoke.Line(m, PointFToPoint(startpoint), PointFToPoint(endpoint), New MCvScalar(0, 255, 0), 3)

End If

CvInvoke.Imshow("m", m)

End Sub

'将PointF转Point方法

Public Shared Function PointFToPoint(ByVal pf As PointF) As Point

Return New Point(CInt(pf.X), CInt(pf.Y))

End Function

输出结果如下图所示：

图11-10 人体关键点连线

下面代码是在上面代码基础上，实现在视频中显示人体关键点的连线。

【代码位置：frmChapter11】Button12_Click、vc_ImageGrabbed

Dim vc As VideoCapture

Dim body_Relations As New List(Of Relation)

'将视频人物标注人体关键点和关键点连线

Private Sub Button12_Click(sender As Object, e As EventArgs) Handles Button12.Click

'18个关键点关联

body_Relations = New List(Of Relation)

body_Relations.Add(New Relation(16, 14))

body_Relations.Add(New Relation(14, 0))

body_Relations.Add(New Relation(17, 15))

body_Relations.Add(New Relation(15, 0))

body_Relations.Add(New Relation(0, 1))

body_Relations.Add(New Relation(1, 2))

body_Relations.Add(New Relation(2, 3))

body_Relations.Add(New Relation(3, 4))

body_Relations.Add(New Relation(1, 5))

body_Relations.Add(New Relation(5, 6))

body_Relations.Add(New Relation(6, 7))

body_Relations.Add(New Relation(1, 8))

body_Relations.Add(New Relation(8, 9))

body_Relations.Add(New Relation(9, 10))

body_Relations.Add(New Relation(1, 11))

body_Relations.Add(New Relation(11, 12))

body_Relations.Add(New Relation(12, 13))

vc = New VideoCapture("C:\learnEmgucv\action.mp4")

If vc.IsOpened = False Then

MessageBox.Show("打开文件失败")

Exit Sub

End If

'添加ImageGrabbed事件

AddHandler vc.ImageGrabbed, AddressOf vc_ImageGrabbed

vc.Start()

End Sub

Private Sub vc_ImageGrabbed(sender As Object, e As EventArgs)

Dim outangle As Double = 0

Dim outpix As Double = 0

Dim nextframe As New Mat

vc.Retrieve(nextframe)

If vc.Get(CapProp.PosFrames) >= vc.Get(CapProp.FrameCount) Then

vc.Stop()

vc.Dispose()

RemoveHandler vc.ImageGrabbed, AddressOf vc_ImageGrabbed

Exit Sub

End If

Dim net As Dnn.Net

net = DnnInvoke.ReadNetFromTensorflow("graph_opt.pb")

Dim blob As Mat

blob = DnnInvoke.BlobFromImage(nextframe, 1.0, New Drawing.Size(360, 360), New MCvScalar(127.5, 127.5, 127.5), True, False)

net.SetInput(blob)

Dim mout As New Mat

mout = net.Forward()

Dim fout(,,,) As Single

fout = mout.GetData()

Dim H As Integer = fout.GetLength(2)

Dim W As Integer = fout.GetLength(3)

Dim lkeypoint As New List(Of Keypoint)

lkeypoint = getMaxPoint(fout)

Dim x, y As Single

For i As Integer = 0 To lkeypoint.Count - 1

x = (lkeypoint(i).p.X / W) * nextframe.Width

y = (lkeypoint(i).p.Y / H) * nextframe.Height

If lkeypoint(i).conf > 0.1 Then

CvInvoke.Circle(nextframe, New Point(x, y), 5, New MCvScalar(0, 0, 255), -1)

End If

Dim startpoint As PointF

Dim startpointx, startpointy As Single

Dim endpoint As PointF

Dim endpointx, endpointy As Single

For Each body_Relation As Relation In body_Relations

startpointx = (lkeypoint(body_Relation.startpoint).p.X / W) * nextframe.Width

startpointy = (lkeypoint(body_Relation.startpoint).p.Y / H) * nextframe.Height

startpoint = New PointF(startpointx, startpointy)

endpointx = (lkeypoint(body_Relation.endpoint).p.X / W) * nextframe.Width

endpointy = (lkeypoint(body_Relation.endpoint).p.Y / H) * nextframe.Height

endpoint = New PointF(endpointx, endpointy)

If lkeypoint(body_Relation.startpoint).conf > 0.15 And lkeypoint(body_Relation.endpoint).conf > 0.15 Then

CvInvoke.Line(nextframe, PointFToPoint(startpoint), PointFToPoint(endpoint), New MCvScalar(0, 255, 0), 4)

End If

ImageBox1.Image = nextframe

Threading.Thread.Sleep(30)

End Sub

输出结果如下图所示：

图11-11 视频中使用人体关键点连线