我的《100-app-challenge》挑战开发100款应用开始啦！持续更新中，可以关注我的专栏了解更多。
之前看到 手势隔空控制，比划就行！| Lazyeat 实现了通过摄像头识别手势来操作电脑，非常有趣，一直想做一个类似的但是更符合我的需求的工具，正好趁这次100-app-challenge，实现了一个浏览器插件------GestureGo手势控制
介绍
GestureGo手势控制是一个能够通过手势操控网页的浏览器插件，在你不方便点击鼠标时，启用此插件，就可轻松使用手指手势 ，实现移动鼠标、滑动、点击网页等操作。
下载
开源地址：github.com/100-app-cha...
已上架edge浏览器扩展商店：microsoftedge.microsoft.com/addons/deta...
截图
移动鼠标
滚动页面
点击页面跳转
暂停/停止识别
实现原理
1、在前端的content-script中，实时收集摄像头内容，转换成图片序列 ，发送至backgorund后端
ts
async captureFrame(video: HTMLVideoElement) {
try {
if (!(video.videoWidth > 0 && video.videoHeight > 0))
return
if (!this.ctx) {
return
}
if (!this.port) {
return
}
if (this.canvas.width !== video.videoWidth || this.canvas.height !== video.videoHeight) {
this.canvas.width = video.videoWidth
this.canvas.height = video.videoHeight
}
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height)
this.ctx!.drawImage(video, 0, 0)
const imageData = this.ctx!.getImageData(0, 0, this.canvas.width, this.canvas.height)
const hexArray = this.imageDataToHexArrayOptimized(imageData)
if (this.isAllBlack(hexArray)) {
console.warn('black frame detected. Skipping...')
return
}
const currentFrame = hexArray
let message: any
if (this.frameCounter % 30 === 0 || !this.previousFrame) {
this.previousFrame = currentFrame
this.frameCounter = 1
message = {
type: 'frame-full',
data: currentFrame,
width: this.canvas.width,
height: this.canvas.height,
timestamp: performance.now(),
}
}
else {
const diffData = this.calculateFrameDiff(currentFrame)
this.previousFrame = currentFrame
if (diffData.length >= 320 * 240) {
this.frameCounter = 1
message = {
type: 'frame-full',
data: currentFrame,
width: this.canvas.width,
height: this.canvas.height,
timestamp: performance.now(),
}
}
else {
this.frameCounter += 1
message = {
type: 'frame-diff',
data: diffData,
width: this.canvas.width,
height: this.canvas.height,
}
}
}
this.port?.postMessage({
type: 'frame',
data: message,
})
}
catch (error) {
console.error('captureFrame error:', error)
}
}2、后端background，接收数据，调用mediapipe识别，判断手势 ，将结果返回content-script
ts
browser.runtime.onConnect.addListener((port) => {
if (port.name === 'handTracking') {
this.activeConnections.add(port)
const canvas = new OffscreenCanvas(320, 240)
const ctx = canvas.getContext('2d', {
willReadFrequently: true,
})
port.onMessage.addListener(async (message: any) => {
if (message.type === 'frame' && this.detector) {
const encodedData = message.data
if (!this.currentFrame || encodedData.type === 'frame-full') {
// 完整帧
this.currentFrame = encodedData.data
canvas.width = encodedData.width
canvas.height = encodedData.height
}
else {
// 差值帧 - 应用变化
const diffData = encodedData.data
for (let i = 0; i < diffData.length; i += 2) {
this.currentFrame[diffData[i]] = diffData[i + 1]
}
}
// 创建 ImageData 并填充数据
let imageData = ctx!.createImageData(canvas.width, canvas.height)
imageData = this.hexArrayToImageDataOptimized(imageData, this.currentFrame!)
ctx!.putImageData(imageData, 0, 0)
// 直接使用 canvas 作为识别输入，而不是 imageData
const result = this.detector.recognize(canvas)
const detection: DetectionResult = {
}
if (result.landmarks && result.handedness) {
for (let i = 0; i < result.landmarks.length; i++) {
const hand: HandInfo = {
landmarks: result.landmarks[i],
handedness: result.handedness[i][0].categoryName as 'Left' | 'Right',
score: result.handedness[i][0].score,
}
if (result.gestures.length > 0) {
hand.categoryName = result.gestures[0][0].categoryName
}
if (hand.handedness === 'Left') {
detection.leftHand = hand
}
else {
detection.rightHand = hand
}
}
}
if (useHorizontalMirror.data.value) {
detection.leftHand?.landmarks.forEach((landmark) => {
landmark.x = 1 - landmark.x
})
detection.rightHand?.landmarks.forEach((landmark) => {
landmark.x = 1 - landmark.x
})
}
port.postMessage({
type: 'handResult',
data: detection,
success: true,
})
const hand = detection.leftHand || detection.rightHand
if (hand) {
const gesture = this.gestureDetector.processHandLandmarks(hand)
port.postMessage({
type: 'gesture',
data: gesture,
success: true,
})
}
}
})
port.onDisconnect.addListener(() => {
this.activeConnections.delete(port)
})
}
}3、contentscript根据返回手势，进行显示和对应的手势操作
ts
watch(handlandMark, (newVal) => {
if (pauseMe.data.value) {
return
}
if (newVal && canvasElement.value) {
const ctx = canvasElement.value.getContext('2d')
if (!ctx) {
return
}
if (!drawingUtils.value) {
drawingUtils.value = new DrawingUtils(ctx)
}
ctx.clearRect(0, 0, 640, 480)
const hand = newVal.rightHand ? newVal.rightHand : newVal.leftHand
if (hand) {
drawingUtils.value.drawConnectors(
hand.landmarks,
GestureRecognizer.HAND_CONNECTIONS,
{
color: '#00ffcc55',
lineWidth: 2,
},
)
drawingUtils.value.drawLandmarks(hand.landmarks, {
color: '#00ffcc99',
lineWidth: 1.2,
})
const w = canvasElement.value.width
const h = canvasElement.value.height
ctx.strokeStyle = '#00ffcc38' // 发光边框颜色
ctx.lineWidth = 4
ctx.strokeRect(w * 0.25, h * 0.25, w * 0.5, h * 0.5)
}
}
})