C# OpencvSharp使用lpd_yunet进行车牌检测

opencv_zoo-main\models\license_plate_detection_yunet中有个lpd_yunet车牌检测的例子，只有python版本的，笔者平时OpencvSharp用的较多，于是想着用OpencvSharp来实现，但是笔者对python不怎么熟悉，经过一番折腾，终于实现了该功能。

效果如下：（改用openvinosharp实现会更快，下面只贴opencvsharp相关代码）

免责声明：车牌图片来源网络，仅供效果测试和学习使用，如有侵权请联系作者删除

模型信息：

全部代码如下：

using System;

using System.Collections.Generic;

using System.Diagnostics;

using System.Linq;

using System.Runtime.InteropServices;

using OpenCvSharp;

using OpenCvSharp.Dnn;

namespace LPD_YuNetTest

{

public class LPD_YuNet : IDisposable

{

private string modelPath;

private Size inputSize;

private float confidenceThreshold;

private float nmsThreshold;

private int topK;

private int keepTopK;

private Backend backendId;

private Target targetId;

private Net model;

private Mat priors;

private readonly string\[\] outputNames = { "loc", "conf", "iou" };

private readonly int\[\]\[\] minSizes = new int\[\]\[\]

{

new int\[\] { 10, 16, 24 },

new int\[\] { 32, 48 },

new int\[\] { 64, 96 },

new int\[\] { 128, 192, 256 }

};

private readonly int\[\] steps = { 8, 16, 32, 64 };

private readonly float\[\] variance = { 0.1f, 0.2f };

public LPD_YuNet(string modelPath, Size inputSize = default, float confThreshold = 0.8f,

float nmsThreshold = 0.3f, int topK = 5000, int keepTopK = 750,

Backend backendId = Backend.OPENCV, Target targetId = Target.CPU)

{

this.modelPath = modelPath;

this.inputSize = inputSize.Width == 0 ? new Size(320, 240) : inputSize;

this.confidenceThreshold = confThreshold;

this.nmsThreshold = nmsThreshold;

this.topK = topK;

this.keepTopK = keepTopK;

this.backendId = backendId;

this.targetId = targetId;

// Load model

model = Net.ReadNet(modelPath);

model.SetPreferableBackend(backendId);

model.SetPreferableTarget(targetId);

// Generate anchors/priorboxes

PriorGen();

}

public void SetInputSize(Size size)

{

inputSize = size;

PriorGen();

}

private Mat Preprocess(Mat image)

{

// 调整为模型期望的输入尺寸

var resized = new Mat();

Cv2.Resize(image, resized, inputSize);

return CvDnn.BlobFromImage(resized, 1.0, inputSize, new Scalar(0, 0, 0), true, false);

}

public Mat\[\] Infer(Mat image)

{

// Preprocess

var inputBlob = Preprocess(image);

// Forward

model.SetInput(inputBlob);

var outputBlobs = outputNames.Select(name => model.Forward(name)).ToArray();

// Postprocess

var results = Postprocess(outputBlobs);

foreach (var blob in outputBlobs)

{

blob.Dispose();

}

return new Mat\[\] { results };

}

private Mat Postprocess(Mat\[\] blob)

{

// Decode

var dets = Decode(blob);

if (dets.Empty())

return new Mat(0, 9, MatType.CV_32F);

// 准备NMS数据

var rects = new List<Rect2d>();

var scores = new List<float>();

var indices = new List<int>();

for (int i = 0; i < dets.Rows; i++)

{

var score = dets.At<float>(i, 8);

if (score < confidenceThreshold)

continue;

// 获取四个点并计算边界矩形

var points = new Point2f $4$ ;

for (int j = 0; j < 4; j++)

{

points $j$ = new Point2f(

dets.At<float>(i, j * 2),

dets.At<float>(i, j * 2 + 1)

);

}

var rect = Cv2.BoundingRect(points);

rects.Add(new Rect2d(rect.X, rect.Y, rect.Width, rect.Height));

scores.Add(score);

indices.Add(i);

}

if (rects.Count == 0)

return new Mat(0, 9, MatType.CV_32F);

// NMS

CvDnn.NMSBoxes(rects, scores, confidenceThreshold, nmsThreshold, out int\[\] keepIndices, 1f, topK);

if (keepIndices.Length == 0)

return new Mat(0, 9, MatType.CV_32F);

// 创建结果矩阵

var result = new Mat(Math.Min(keepIndices.Length, keepTopK), 9, MatType.CV_32F);

for (int i = 0; i < Math.Min(keepIndices.Length, keepTopK); i++)

{

var originalIndex = indices $keepIndices\[i$ ];

dets.Row(originalIndex).CopyTo(result.Row(i));

}

return result;

}

private void PriorGen()

{

int w = inputSize.Width;

int h = inputSize.Height;

var featureMap2th = new int\[\] { ((h + 1) / 2) / 2, ((w + 1) / 2) / 2 };

var featureMap3th = new int\[\] { featureMap2th $0$ / 2, featureMap2th $1$ / 2 };

var featureMap4th = new int\[\] { featureMap3th $0$ / 2, featureMap3th $1$ / 2 };

var featureMap5th = new int\[\] { featureMap4th $0$ / 2, featureMap4th $1$ / 2 };

var featureMap6th = new int\[\] { featureMap5th $0$ / 2, featureMap5th $1$ / 2 };

var featureMaps = new int\[\]\[\] { featureMap3th, featureMap4th, featureMap5th, featureMap6th };

var priorsList = new List<float\[\]>();

for (int k = 0; k < featureMaps.Length; k++)

{

var minSizesK = minSizes $k$ ;

var f = featureMaps $k$ ;

var step = steps $k$ ;

for (int i = 0; i < f $0$ ; i++)

{

for (int j = 0; j < f $1$ ; j++)

{

foreach (var minSize in minSizesK)

{

float s_kx = minSize / (float)w;

float s_ky = minSize / (float)h;

float cx = (j + 0.5f) * step / w;

float cy = (i + 0.5f) * step / h;

priorsList.Add(new float\[\] { cx, cy, s_kx, s_ky });

}

priors?.Dispose();

priors = new Mat(priorsList.Count, 4, MatType.CV_32F);

for (int i = 0; i < priorsList.Count; i++)

{

priors.Set<float>(i, 0, priorsList $i$ $0$ );

priors.Set<float>(i, 1, priorsList $i$ $1$ );

priors.Set<float>(i, 2, priorsList $i$ $2$ );

priors.Set<float>(i, 3, priorsList $i$ $3$ );

}

private Mat Decode(Mat\[\] blob)

{

var loc = blob $0$ ; // $4385, 14$

var conf = blob $1$ ; // $4385, 2$

var iou = blob $2$ ; // $4385, 1$

int numPriors = loc.Rows;

var dets = new Mat(numPriors, 9, MatType.CV_32F);

// 使用指针操作提高性能

unsafe

{

float* locPtr = (float*)loc.Data;

float* confPtr = (float*)conf.Data;

float* iouPtr = (float*)iou.Data;

float* priorsPtr = (float*)priors.Data;

float* detsPtr = (float*)dets.Data;

for (int i = 0; i < numPriors; i++)

{

// 计算分数

float clsScore = confPtr $i \* 2 + 1$ ; // 第二个类的分数

float iouScore = Math.Max(0, Math.Min(1, iouPtr $i$ ));

float score = (float)Math.Sqrt(clsScore * iouScore);

// 获取先验框参数

float cx = priorsPtr $i \* 4$ ;

float cy = priorsPtr $i \* 4 + 1$ ;

float s_kx = priorsPtr $i \* 4 + 2$ ;

float s_ky = priorsPtr $i \* 4 + 3$ ;

// 计算四个角点

float* locRow = locPtr + i * 14;

// 点1 (左上)

detsPtr $i \* 9$ = (cx + locRow $4$ * variance $0$ * s_kx) * inputSize.Width;

detsPtr $i \* 9 + 1$ = (cy + locRow $5$ * variance $0$ * s_ky) * inputSize.Height;

// 点2 (右上)

detsPtr $i \* 9 + 2$ = (cx + locRow $6$ * variance $0$ * s_kx) * inputSize.Width;

detsPtr $i \* 9 + 3$ = (cy + locRow $7$ * variance $0$ * s_ky) * inputSize.Height;

// 点3 (右下)

detsPtr $i \* 9 + 4$ = (cx + locRow $10$ * variance $0$ * s_kx) * inputSize.Width;

detsPtr $i \* 9 + 5$ = (cy + locRow $11$ * variance $0$ * s_ky) * inputSize.Height;

// 点4 (左下)

detsPtr $i \* 9 + 6$ = (cx + locRow $12$ * variance $0$ * s_kx) * inputSize.Width;

detsPtr $i \* 9 + 7$ = (cy + locRow $13$ * variance $0$ * s_ky) * inputSize.Height;

// 分数

detsPtr $i \* 9 + 8$ = score;

}

return dets;

}

public void Dispose()

{

model?.Dispose();

priors?.Dispose();

}

public class LPD_YuNetDemo

{

public static Mat Visualize(Mat image, Mat dets, double scaleX = 1.0, double scaleY = 1.0, Scalar lineColor = default, Scalar textColor = default, double? fps = null)

{

lineColor = lineColor == default ? new Scalar(0, 255, 0) : lineColor;

textColor = textColor == default ? new Scalar(0, 0, 255) : textColor;

Mat output = image.Clone();

if (fps.HasValue)

{

Cv2.PutText(output, $"FPS: {fps.Value:F2}", new Point(10, 30),

HersheyFonts.HersheySimplex, 0.7, textColor, 2);

}

if (dets.Empty())

return output;

for (int i = 0; i < dets.Rows; i++)

{

var bbox = new float $8$ ;

Marshal.Copy(dets.Row(i).Data,bbox,0,8);

//dets.Row(i).GetArray(out bbox);

// 缩放坐标到原始图像尺寸

var points = new Point\[\]

{

new Point((int)(bbox $0$ * scaleX), (int)(bbox $1$ * scaleY)),

new Point((int)(bbox $2$ * scaleX), (int)(bbox $3$ * scaleY)),

new Point((int)(bbox $4$ * scaleX), (int)(bbox $5$ * scaleY)),

new Point((int)(bbox $6$ * scaleX), (int)(bbox $7$ * scaleY))

};

// 绘制车牌边框

for (int j = 0; j < 4; j++)

{

Cv2.Line(output, points $j$ , points $(j + 1) % 4$ , lineColor, 2);

}

// 显示置信度

float score = dets.At<float>(i, 8);

Cv2.PutText(output, $"{score:F2}", points $0$ ,

HersheyFonts.HersheySimplex, 0.5, textColor, 1);

}

return output;

}

public static void Main2()

{

string modelPath = "D:\\opencvsharp\\1\\opencv_zoo-main\\opencv_zoo-main\\models\\license_plate_detection_yunet\\license_plate_detection_lpd_yunet_2023mar.onnx";

if (!System.IO.File.Exists(modelPath))

{

Console.WriteLine($"Model file not found: {modelPath}");

return;

}

var inputSize = new Size(320, 240);

var model = new LPD_YuNet(

modelPath: modelPath,

inputSize: inputSize,

confThreshold: 0.6f, // 降低阈值提高检测率

nmsThreshold: 0.3f,

topK: 1000, // 减少处理的数量

keepTopK: 10,

backendId: 0, // OpenCV backend

targetId: 0 // CPU

);

var choice = "1";

if (choice == "1")

{

// 摄像头模式

var cap = new VideoCapture(0);

if (!cap.IsOpened())

{

Console.WriteLine("Cannot open camera!");

return;

}

cap.Set(VideoCaptureProperties.FrameWidth, 640);

cap.Set(VideoCaptureProperties.FrameHeight, 480);

var stopwatch = new Stopwatch();

int frameCount = 0;

double fps = 0;

while (true)

{

var frame = new Mat();

if (!cap.Read(frame) || frame.Empty())

{

Console.WriteLine("No frames grabbed!");

break;

}

stopwatch.Start();

// 推理

var results = model.Infer(frame);

var result = results $0$ ;

stopwatch.Stop();

frameCount++;

// 计算FPS

if (stopwatch.Elapsed.TotalSeconds >= 1.0)

{

fps = frameCount / stopwatch.Elapsed.TotalSeconds;

frameCount = 0;

stopwatch.Restart();

}

// 计算缩放比例

double scaleX = (double)frame.Width / inputSize.Width;

double scaleY = (double)frame.Height / inputSize.Height;

// 可视化结果

var displayFrame = Visualize(frame, result, scaleX, scaleY, fps: fps);

Cv2.ImShow("LPD-YuNet Demo", displayFrame);

// 显示检测数量

Console.WriteLine($"Detected: {result.Rows} license plates, FPS: {fps:F2}");

if (Cv2.WaitKey(1) == 27) // ESC键退出

break;

}

else

{

// 图像文件模式

Console.Write("Enter image path: ");

var imagePath = Console.ReadLine();

if (!System.IO.File.Exists(imagePath))

{

Console.WriteLine($"Image file not found: {imagePath}");

return;

}

var image = Cv2.ImRead(imagePath);

if (image.Empty())

{

Console.WriteLine("Failed to load image!");

return;

}

// 设置模型输入尺寸为图像尺寸

model.SetInputSize(new Size(image.Width, image.Height));

var stopwatch = Stopwatch.StartNew();

var results = model.Infer(image);

stopwatch.Stop();

var result = results $0$ ;

Console.WriteLine($"Inference time: {stopwatch.ElapsedMilliseconds}ms");

Console.WriteLine($"Detected {result.Rows} license plates");

// 可视化结果

var resultImage = Visualize(image, result);

Cv2.ImShow("Detection Result", resultImage);

Cv2.WaitKey(0);

// 保存结果

Cv2.ImWrite("detection_result.jpg", resultImage);

Console.WriteLine("Result saved to detection_result.jpg");

}