EmguCV学习笔记 VB.Net 11.6 图像分割

EmguCV是一个基于OpenCV的开源免费的跨平台计算机视觉库,它向C#和VB.NET开发者提供了OpenCV库的大部分功能。

教程VB.net版本请访问： EmguCV学习笔记 VB.Net 目录-CSDN博客

教程C#版本请访问： EmguCV学习笔记 C# 目录-CSDN博客

笔者的博客网址：https://blog.csdn.net/uruseibest

教程配套文件及相关说明以及如何获得pdf教程和代码，请移步： EmguCV学习笔记

学习VB.Net知识，请移步： vb.net 教程目录_vb中如何用datagridview-CSDN博客

学习C#知识，请移步： C# 教程目录_c#教程目录-CSDN博客

11.6 图像分割

11.6.1 语义分割Fcn

FCN（Fully Convolutional Network）是一种流行的语义分割算法，它通过将传统的卷积神经网络转化为全卷积网络来实现像素级别的语义分割，将输入图像传入网络中，得到每个像素的标签，并根据标签来分割出不同的物体。FCN在语义分割任务中表现优秀，可以实现像素级别的精细分割，被广泛应用于自动驾驶、智能监控等领域。

【代码位置：frmChapter11】Button5_Click、getColorTable、showColorTable

'语义分割fcn

Private Sub Button5_Click(sender As Object, e As EventArgs) Handles Button5.Click

'对象分类，object_detection_classes_pascal_voc.txt文件提供了21类对象（含background）

Dim classnames() As String

classnames = File.ReadAllLines("C:\learnEmgucv\fcn\object_detection_classes_pascal_voc.txt")

'获得输出颜色表

Dim colorTable() As Bgr = getColorTable()

'显示颜色表及对应对象名称

Call showColorTable(classnames, colorTable)

'需要测试的图像文件

Dim m As New Mat("C:\learnEmgucv\dnntest1.jpg", ImreadModes.Color)

Dim hm As Single = m.Height

Dim wm As Single = m.Width

Dim net As Dnn.Net

net = DnnInvoke.ReadNetFromCaffe("C:\learnEmgucv\fcn\fcn8s-heavy-pascal.prototxt",

"C:\learnEmgucv\fcn\fcn8s-heavy-pascal.caffemodel"

)

Dim mcopy As New Mat

CvInvoke.Resize(m, mcopy, New Drawing.Size(500, 500))

Dim blob As Mat

blob = DnnInvoke.BlobFromImage(mcopy, 1, New Size(500, 500),

New MCvScalar(0， 0， 0), False, False)

net.SetInput(blob)

Dim mout As New Mat

mout = net.Forward()

Dim fout(,,,) As Single

fout = mout.GetData()

'通道数=21，即21类对象

Dim chan As Integer = fout.GetLength(1)

'高度=500

Dim row As Integer = fout.GetLength(2)

'宽度=500

Dim col As Integer = fout.GetLength(3)

'记录21个通道对应坐标点的最大值

Dim matrMaxValue As New Matrix(Of Single)(New Size(row, col))

matrMaxValue.SetZero()

''记录对应通道号

'Dim matrMaxChan As New Matrix(Of Single)(New Size(row, col))

'matrMaxChan.SetZero()

'记录对应颜色

Dim imgOut As New Image(Of Bgr, Byte)(col, row)

imgOut.SetZero()

'遍历21个通道

For c As Integer = 0 To chan - 1

For h As Integer = 0 To row - 1

'遍历高度和宽度，获得对应坐标的值

For w As Integer = 0 To col - 1

'比较最大值

If fout(0, c, h, w) > matrMaxValue(h, w) Then

'取得最大值

matrMaxValue(h, w) = fout(0, c, h, w)

''取得通道号

'matrMaxChan(h, w) = c

'最重要的是获取通道对应颜色表的值

imgOut(h, w) = colorTable(c)

End If

'显示输出的图像

'ImageBox1.Image = imgOut.Mat

'设置掩膜

Dim mask As New Mat

mask = imgOut.Mat

'掩膜大小必须和源图像一致

CvInvoke.Resize(mask, mask, m.Size)

'将上面的输出图像和源图像叠加

Dim mFinalOut As New Mat

CvInvoke.AddWeighted(m, 0.3, mask, 0.7, 0, mFinalOut)

ImageBox1.Image = mFinalOut

End Sub

'为了更好地观察，这里没有使用随机颜色

Private Function getColorTable() As Bgr()

Dim newColors(20) As Bgr

newColors(0) = New Bgr(0, 0, 0) 'background

newColors(1) = New Bgr(128, 0, 0) 'aeroplane

newColors(2) = New Bgr(0, 128, 0) 'bicycle

newColors(3) = New Bgr(128, 128, 0) 'bird

newColors(4) = New Bgr(0, 0, 128) 'boat

newColors(5) = New Bgr(128, 0, 128) 'bottle

newColors(6) = New Bgr(0, 128, 128) 'bus

newColors(7) = New Bgr(128, 128, 128) 'car

newColors(8) = New Bgr(255, 0, 0) 'cat

newColors(9) = New Bgr(0, 255, 0) 'chair

newColors(10) = New Bgr(0, 0, 255) 'cow

newColors(11) = New Bgr(255, 255, 0) 'diningtable

newColors(12) = New Bgr(64, 0, 128) 'dog

newColors(13) = New Bgr(192, 0, 128) 'horse

newColors(14) = New Bgr(64, 128, 128) 'motorbike

newColors(15) = New Bgr(192, 128, 128) 'person

newColors(16) = New Bgr(0, 64, 0) 'pottedplant

newColors(17) = New Bgr(128, 64, 64) 'sheep

newColors(18) = New Bgr(0, 192, 0) 'sofa

newColors(19) = New Bgr(128, 192, 0) 'train

newColors(20) = New Bgr(0, 64, 128) 'tvmonitor

Return newColors

End Function

'显示颜色表及对象名称

'参数1：对象名称的字符串数组

'参数2：bgr颜色数组

Private Sub showColorTable(ByVal names() As String, ByVal colors() As Bgr)

Dim lstmoutV As New List(Of Mat)

For i As Integer = 0 To 20

Dim moutV As New Mat(40, 200, DepthType.Cv8U, 3)

moutV.SetTo(New MCvScalar(colors(i).Blue, colors(i).Green, colors(i).Red))

CvInvoke.PutText(moutV, names(i), New Point(20, 25), FontFace.HersheyTriplex, 0.4, New MCvScalar(255, 255, 255))

lstmoutV.Add(moutV)

Dim mout As New Mat

'垂直方向拼接

CvInvoke.VConcat(lstmoutV.ToArray, mout)

CvInvoke.Imshow("colortable", mout)

End Sub

输出结果如下图所示：

图11-4 FCN分割获得不同对象区域

11.6.2 实例分割 MASK RCNN

Mask RCNN是一种基于Faster R-CNN的实例分割算法，它是一种联合目标检测和语义分割的方法，能够同时检测图像中的对象并得到每个对象的位置、类别和像素级别的分割结果。Mask RCNN在实例分割任务中表现优秀，是目前最先进的实例分割算法之一，被广泛应用于自动驾驶、智能监控等领域。

【代码位置：frmChapter11】Button6_Click、getRadomColor

'实例分割 mask rcnn

Private Sub Button6_Click(sender As Object, e As EventArgs) Handles Button6.Click

'对象分类，object_detection_classes_coco.txt文件提供了90类对象（含background）

Dim classnames() As String

classnames = File.ReadAllLines("C:\learnEmgucv\maskrcnn\object_detection_classes_coco.txt")

'需要测试的图像文件

Dim m As New Mat("C:\learnEmgucv\dnntest.jpg", ImreadModes.Color)

Dim hm As Single = m.Height

Dim wm As Single = m.Width

Dim net As Dnn.Net

net = DnnInvoke.ReadNetFromTensorflow("C:\learnEmgucv\maskrcnn\frozen_inference_graph.pb",

"C:\learnEmgucv\maskrcnn\mask_rcnn_inception_v2_coco_2018_01_28.pbtxt"

)

Dim mcopy As New Mat

mcopy = m.Clone

Dim blob As Mat

blob = DnnInvoke.BlobFromImage(mcopy, 1, New Size(500, 500),

New MCvScalar(0， 0， 0), False, False)

net.SetInput(blob)

Dim names(1) As String

names（0） = "detection_out_final"

names（1） = "detection_masks"

'返回的mout包含两个mat

Dim mout As New VectorOfMat

net.Forward(mout, names)

'第一个mat标识返回的置信度候选框，是一个四维数组

Dim moutBox As New Mat

moutBox = mout(0)

'返回维度：

'第1维：1，

'第2维：1，

'第3维：100，100个候选置信矩形框

'第4维：7，0：?;1:对应类别；2：置信度；3-6：候选框位置（源图像百分比）

Dim foutBox(,,,) As Single

foutBox = moutBox.GetData()

'第二个mat标识返回的掩膜，是一个四维数组

Dim moutMask As New Mat

moutMask = mout(1)

'返回维度：

'第1维：100，100个对象对应的100个掩膜

'第2维：90，对象的置信度

'第3维：15，掩膜高度

'第4维：15，掩膜宽度

Dim foutMask(,,,) As Single

foutMask = moutMask.GetData()

Dim maskH As Integer = 15 'foutMask.GetLength(2)

Dim maskW As Integer = 15 'foutMask.GetLength(3)

'新建Mat，用来在这上面绘制掩膜

Dim mbg As New Mat(New Size(m.Width, m.Height), DepthType.Cv8U, 3)

mbg.SetTo(New MCvScalar(0, 0, 0))

For i As Integer = 0 To 100 - 1

'置信度

Dim conf As Single = foutBox(0, 0, i, 2)

'当置信度满足时

If conf > 0.53 Then

'对应检测对象的序号

Dim objID As Integer = foutBox(0, 0, i, 1)

Dim ltX As Single = foutBox(0, 0, i, 3) * wm '左上角X

Dim ltY As Single = foutBox(0, 0, i, 4) * hm '左上角Y

Dim rbX As Single = foutBox(0, 0, i, 5) * wm '右下角X

Dim rbY As Single = foutBox(0, 0, i, 6) * hm '右下角Y

Dim w As Single = rbX - ltX '宽度

Dim h As Single = rbY - ltY '高度

'绘制包围矩形框

CvInvoke.Rectangle(m, New Rectangle(ltX, ltY, w, h), New MCvScalar(0, 0, 255), 1)

'绘制对象名称

CvInvoke.PutText(m, classnames(objID), New Point(ltX, ltY - 10), FontFace.HersheyTriplex, 0.3, New MCvScalar(255, 0, 0))

'开始处理掩膜

Dim mmask As New Mat()

'掩膜大小为15*15，对应foutMask最后两个维度

Dim bmask(14, 14) As Single

For j As Integer = 0 To 14

For k As Integer = 0 To 14

bmask(j, k) = foutMask(i, objID, j, k)

'将数组转为Mat

Dim matrmask As New Matrix(Of Single)(bmask)

mmask = matrmask.Mat

'大小放大与对应包围矩形框一致

CvInvoke.Resize(mmask, mmask, New Drawing.Size(w, h))

'二值化

CvInvoke.Threshold(mmask, mmask, 0.3, 255, ThresholdType.Binary)

'由于本身是CV32F，需要处理为CV8U，才能使用FindContours

mmask.ConvertTo(mmask, DepthType.Cv8U)

'定义关注区域，当修改关注区域时，也就修改了源图像

Dim mRoi As New Mat(mbg, New Rectangle(ltX, ltY, w, h))

'查找轮廓

Dim contours As New VectorOfVectorOfPoint

CvInvoke.FindContours(mmask, contours, Nothing, RetrType.External, ChainApproxMethod.ChainApproxSimple)

'填充轮廓

CvInvoke.DrawContours(mRoi, contours, -1, getRadomColor(), -1)

End If

Dim mresult As New Mat

'加法

CvInvoke.AddWeighted(m, 1, mbg, 0.4, 0, mresult)

ImageBox1.Image = mresult

End Sub

'获得随机颜色

Private Function getRadomColor() As MCvScalar

Dim rd As New Random(Now.Millisecond)

Dim r, g, b As Byte

b = rd.Next(256)

g = rd.Next(256)

r = rd.Next(256)

Return New MCvScalar(b, g, r)

End Function

输出结果如下图所示：

图11-5 Mask RCNN 分割得到对象区域和类别