概述
机器学习正在革命性地改变3D交互方式。本节将探索如何集成TensorFlow.js进行实时3D姿态估计，并使用估计结果驱动Three.js中的骨骼动画，实现从2D视频到3D角色的自然映射。
ML-3D系统架构：
摄像头输入 姿态估计模型 2D关键点检测 3D姿态重建 骨骼映射 动画驱动 运动重定向 3D角色动画 实时渲染 动作捕捉
核心原理
姿态估计算法对比
| 模型 | 精度 | 速度 | 3D支持 | 适用场景 |
|---|---|---|---|---|
| PoseNet | 中等 | 快 | 否 | 实时应用 |
| MoveNet | 高 | 很快 | 否 | 运动分析 |
| MediaPipe Pose | 很高 | 快 | 是 | 专业应用 |
| BlazePose | 极高 | 中等 | 是 | 医疗健身 |
骨骼映射原理
javascript
// 关键点映射配置
class PoseMapper {
static BODY_CONNECTIONS = [
// 身体主干
['left_shoulder', 'right_shoulder'],
['left_shoulder', 'left_hip'],
['right_shoulder', 'right_hip'],
['left_hip', 'right_hip'],
// 左臂
['left_shoulder', 'left_elbow'],
['left_elbow', 'left_wrist'],
// 右臂
['right_shoulder', 'right_elbow'],
['right_elbow', 'right_wrist'],
// 左腿
['left_hip', 'left_knee'],
['left_knee', 'left_ankle'],
// 右腿
['right_hip', 'right_knee'],
['right_knee', 'right_ankle']
];
static mapToBoneRotations(keypoints) {
const rotations = {};
for (const [start, end] of this.BODY_CONNECTIONS) {
const startPoint = keypoints.find(k => k.name === start);
const endPoint = keypoints.find(k => k.name === end);
if (startPoint && endPoint) {
const direction = new THREE.Vector3()
.subVectors(endPoint.position, startPoint.position)
.normalize();
rotations[`${start}_to_${end}`] = this.vectorToQuaternion(direction);
}
}
return rotations;
}
static vectorToQuaternion(direction) {
// 将方向向量转换为四元数
const up = new THREE.Vector3(0, 1, 0);
const quaternion = new THREE.Quaternion();
quaternion.setFromUnitVectors(up, direction);
return quaternion;
}
}完整代码实现
实时姿态估计系统
vue
<template>
<div class="pose-estimation-container">
<!-- 视频输入和3D输出 -->
<div class="main-view">
<!-- 视频源 -->
<div class="video-section">
<video ref="videoElement"
class="video-feed"
autoplay
playsinline
@loadedmetadata="onVideoReady">
</video>
<canvas ref="poseCanvas" class="pose-overlay"></canvas>
<!-- 视频控制 -->
<div class="video-controls">
<button @click="toggleCamera" class="control-button">
{{ isCameraActive ? '📷 停止' : '📷 开始' }}
</button>
<button @click="toggleMirror" class="control-button">
{{ mirrorMode ? '🔁 关闭镜像' : '🔁 开启镜像' }}
</button>
<button @click="takeSnapshot" class="control-button">
📸 截图
</button>
</div>
</div>
<!-- 3D角色 -->
<div class="character-section">
<canvas ref="characterCanvas" class="character-canvas"></canvas>
<!-- 角色控制 -->
<div class="character-controls">
<button @click="toggleCharacter" class="control-button">
{{ showCharacter ? '👤 隐藏角色' : '👤 显示角色' }}
</button>
<button @click="resetPose" class="control-button">
🔄 重置姿势
</button>
<button @click="calibratePose" class="control-button">
🎯 校准
</button>
</div>
</div>
</div>
<!-- 控制面板 -->
<div class="control-panel">
<div class="panel-section">
<h3>🤖 模型设置</h3>
<div class="model-config">
<div class="config-group">
<label>姿态估计模型</label>
<select v-model="selectedModel" @change="loadModel">
<option value="movenet">MoveNet (轻量级)</option>
<option value="posenet">PoseNet (平衡)</option>
<option value="blazepose">BlazePose (高精度)</option>
</select>
</div>
<div class="config-group">
<label>模型复杂度</label>
<select v-model="modelComplexity">
<option value="lite">Lite (最快)</option>
<option value="full">Full (平衡)</option>
<option value="heavy">Heavy (最准)</option>
</select>
</div>
<div class="config-group">
<label>检测置信度</label>
<input type="range" v-model="minPoseConfidence" min="0.1" max="1" step="0.1">
<span>{{ minPoseConfidence }}</span>
</div>
<div class="config-group">
<label>关键点置信度</label>
<input type="range" v-model="minKeypointConfidence" min="0.1" max="1" step="0.1">
<span>{{ minKeypointConfidence }}</span>
</div>
</div>
</div>
<div class="panel-section">
<h3>🎭 角色设置</h3>
<div class="character-config">
<div class="config-group">
<label>角色类型</label>
<select v-model="characterType" @change="loadCharacter">
<option value="stickman">简笔画人</option>
<option value="humanoid">人形角色</option>
<option value="robot">机器人</option>
<option value="custom">自定义</option>
</select>
</div>
<div class="config-group">
<label>骨骼粗细</label>
<input type="range" v-model="boneThickness" min="0.1" max="2" step="0.1">
<span>{{ boneThickness }}</span>
</div>
<div class="config-group">
<label>关节大小</label>
<input type="range" v-model="jointSize" min="0.1" max="1" step="0.1">
<span>{{ jointSize }}</span>
</div>
<div class="config-group">
<label>平滑系数</label>
<input type="range" v-model="smoothingFactor" min="0" max="1" step="0.1">
<span>{{ smoothingFactor }}</span>
</div>
</div>
</div>
<div class="panel-section">
<h3>📊 性能监控</h3>
<div class="performance-monitor">
<div class="performance-item">
<span>检测帧率:</span>
<span>{{ detectionFPS }} FPS</span>
</div>
<div class="performance-item">
<span>渲染帧率:</span>
<span>{{ renderFPS }} FPS</span>
</div>
<div class="performance-item">
<span>检测延迟:</span>
<span>{{ detectionTime }}ms</span>
</div>
<div class="performance-item">
<span>关键点数量:</span>
<span>{{ keypointCount }}</span>
</div>
<div class="performance-item">
<span>跟踪质量:</span>
<span :class="trackingQualityClass">{{ trackingQuality }}</span>
</div>
</div>
</div>
<div class="panel-section">
<h3>🎮 动作捕捉</h3>
<div class="mocap-controls">
<div class="control-group">
<label>
<input type="checkbox" v-model="enableRecording">
录制动作
</label>
</div>
<div class="control-group">
<label>
<input type="checkbox" v-model="enablePlayback">
回放动作
</label>
</div>
<div class="recording-controls" v-if="enableRecording">
<button @click="startRecording" class="control-button" :disabled="isRecording">
● 开始录制
</button>
<button @click="stopRecording" class="control-button" :disabled="!isRecording">
■ 停止录制
</button>
<button @click="saveRecording" class="control-button" :disabled="!recordedData.length">
💾 保存数据
</button>
</div>
<div class="playback-controls" v-if="enablePlayback">
<button @click="loadRecording" class="control-button">
📁 加载数据
</button>
<button @click="playRecording" class="control-button" :disabled="isPlaying">
▶️ 播放
</button>
<button @click="pauseRecording" class="control-button" :disabled="!isPlaying">
⏸️ 暂停
</button>
</div>
</div>
</div>
<div class="panel-section">
<h3>🔧 高级设置</h3>
<div class="advanced-settings">
<div class="setting-group">
<label>
<input type="checkbox" v-model="enable3DReconstruction">
3D姿态重建
</label>
</div>
<div class="setting-group">
<label>
<input type="checkbox" v-model="enableSkeletonSmoothing">
骨骼平滑
</label>
</div>
<div class="setting-group">
<label>
<input type="checkbox" v-model="showKeypoints">
显示关键点
</label>
</div>
<div class="setting-group">
<label>
<input type="checkbox" v-model="showSkeleton">
显示骨骼
</label>
</div>
<div class="setting-group">
<label>镜像模式</label>
<select v-model="mirrorMode">
<option value="none">无镜像</option>
<option value="horizontal">水平镜像</option>
<option value="vertical">垂直镜像</option>
</select>
</div>
</div>
</div>
</div>
<!-- 关键点信息面板 -->
<div v-if="showKeypointPanel" class="keypoint-panel">
<div class="panel-header">
<h4>关键点信息</h4>
<button @click="showKeypointPanel = false" class="close-button">×</button>
</div>
<div class="keypoint-list">
<div v-for="keypoint in currentKeypoints"
:key="keypoint.name"
:class="['keypoint-item', { active: keypoint.score >= minKeypointConfidence }]">
<span class="keypoint-name">{{ keypoint.name }}</span>
<span class="keypoint-score">{{ (keypoint.score * 100).toFixed(1) }}%</span>
<span class="keypoint-position">
({{ keypoint.position.x.toFixed(1) }}, {{ keypoint.position.y.toFixed(1) }})
</span>
</div>
</div>
</div>
<!-- 加载状态 -->
<div v-if="isLoading" class="loading-overlay">
<div class="loading-content">
<div class="spinner"></div>
<h3>加载模型中...</h3>
<p>{{ loadingProgress }}%</p>
</div>
</div>
</div>
</template>
<script>
import { onMounted, onUnmounted, ref, reactive, computed } from 'vue';
import * as THREE from 'three';
import * as poseDetection from '@tensorflow-models/pose-detection';
import '@tensorflow/tfjs-backend-webgl';
// 姿态估计管理器
class PoseEstimationManager {
constructor() {
this.detector = null;
this.isInitialized = false;
this.lastDetectionTime = 0;
}
async initialize(modelType = 'movenet', complexity = 'lite') {
try {
// 设置TensorFlow.js后端
await tf.setBackend('webgl');
// 创建检测器配置
const detectorConfig = this.getDetectorConfig(modelType, complexity);
// 加载模型
this.detector = await poseDetection.createDetector(
modelType === 'blazepose' ? poseDetection.SupportedModels.BlazePose :
modelType === 'posenet' ? poseDetection.SupportedModels.PoseNet :
poseDetection.SupportedModels.MoveNet,
detectorConfig
);
this.isInitialized = true;
console.log('姿态估计模型加载成功');
} catch (error) {
console.error('模型加载失败:', error);
throw error;
}
}
getDetectorConfig(modelType, complexity) {
const baseConfig = {
modelType: complexity.toUpperCase(),
enableSmoothing: true,
minPoseScore: 0.25
};
switch (modelType) {
case 'movenet':
return {
...baseConfig,
modelType: complexity === 'heavy' ? 'THUNDER' : 'LIGHTNING'
};
case 'blazepose':
return {
...baseConfig,
runtime: 'tfjs',
enableSmoothing: true,
modelType: complexity === 'heavy' ? 'full' : 'lite'
};
case 'posenet':
return {
...baseConfig,
architecture: 'MobileNetV1',
outputStride: 16,
inputResolution: { width: 640, height: 480 },
multiplier: complexity === 'heavy' ? 1.0 : 0.75
};
default:
return baseConfig;
}
}
async estimatePoses(videoElement, minScore = 0.3) {
if (!this.detector || !this.isInitialized) {
throw new Error('检测器未初始化');
}
const startTime = performance.now();
try {
const poses = await this.detector.estimatePoses(videoElement, {
maxPoses: 1,
flipHorizontal: false
});
const detectionTime = performance.now() - startTime;
// 过滤低置信度的姿态
const validPoses = poses.filter(pose => pose.score >= minScore);
return {
poses: validPoses,
detectionTime,
keypointCount: validPoses.length > 0 ? validPoses[0].keypoints.length : 0
};
} catch (error) {
console.error('姿态估计失败:', error);
return { poses: [], detectionTime: 0, keypointCount: 0 };
}
}
dispose() {
if (this.detector) {
this.detector.dispose();
}
this.isInitialized = false;
}
}
// 3D角色管理器
class CharacterManager {
constructor(renderer, scene, camera) {
this.renderer = renderer;
this.scene = scene;
this.camera = camera;
this.character = null;
this.bones = new Map();
this.joints = new Map();
this.setupScene();
}
setupScene() {
// 基础照明
const ambientLight = new THREE.AmbientLight(0x404040, 0.6);
this.scene.add(ambientLight);
const directionalLight = new THREE.DirectionalLight(0xffffff, 0.8);
directionalLight.position.set(10, 10, 5);
this.scene.add(directionalLight);
// 网格地面
const gridHelper = new THREE.GridHelper(10, 10);
this.scene.add(gridHelper);
}
// 创建简笔画角色
createStickFigure() {
this.clearCharacter();
const characterGroup = new THREE.Group();
characterGroup.name = 'stick_figure';
// 创建骨骼材质
const boneMaterial = new THREE.MeshBasicMaterial({
color: 0x00ff00,
transparent: true,
opacity: 0.8
});
const jointMaterial = new THREE.MeshBasicMaterial({
color: 0xff0000,
transparent: true,
opacity: 0.9
});
// 定义骨骼连接
const boneConnections = [
// 身体主干
{ name: 'spine', start: 'hip', end: 'shoulder_center', radius: 0.1 },
// 左臂
{ name: 'left_upper_arm', start: 'shoulder_center', end: 'left_elbow', radius: 0.08 },
{ name: 'left_lower_arm', start: 'left_elbow', end: 'left_wrist', radius: 0.06 },
// 右臂
{ name: 'right_upper_arm', start: 'shoulder_center', end: 'right_elbow', radius: 0.08 },
{ name: 'right_lower_arm', start: 'right_elbow', end: 'right_wrist', radius: 0.06 },
// 左腿
{ name: 'left_upper_leg', start: 'hip', end: 'left_knee', radius: 0.1 },
{ name: 'left_lower_leg', start: 'left_knee', end: 'left_ankle', radius: 0.08 },
// 右腿
{ name: 'right_upper_leg', start: 'hip', end: 'right_knee', radius: 0.1 },
{ name: 'right_lower_leg', start: 'right_knee', end: 'right_ankle', radius: 0.08 }
];
// 创建骨骼
boneConnections.forEach(connection => {
const bone = this.createBone(connection.start, connection.end, connection.radius, boneMaterial);
characterGroup.add(bone);
this.bones.set(connection.name, bone);
});
// 创建关节
const jointPoints = [
'hip', 'shoulder_center', 'left_elbow', 'left_wrist',
'right_elbow', 'right_wrist', 'left_knee', 'left_ankle',
'right_knee', 'right_ankle'
];
jointPoints.forEach(jointName => {
const joint = this.createJoint(jointName, 0.15, jointMaterial);
characterGroup.add(joint);
this.joints.set(jointName, joint);
});
this.scene.add(characterGroup);
this.character = characterGroup;
return characterGroup;
}
createBone(startJoint, endJoint, radius, material) {
// 创建圆柱体作为骨骼
const boneGeometry = new THREE.CylinderGeometry(radius, radius, 1, 8);
const bone = new THREE.Mesh(boneGeometry, material);
// 初始位置和方向
bone.userData = { startJoint, endJoint };
bone.rotation.order = 'YXZ';
return bone;
}
createJoint(name, radius, material) {
const jointGeometry = new THREE.SphereGeometry(radius, 8, 6);
const joint = new THREE.Mesh(jointGeometry, material);
joint.name = name;
return joint;
}
// 更新角色姿态
updatePose(keypoints, smoothing = 0.5) {
if (!this.character || !keypoints || keypoints.length === 0) return;
// 将2D关键点映射到3D空间
const jointPositions = this.mapKeypointsTo3D(keypoints);
// 更新关节位置
for (const [jointName, position] of Object.entries(jointPositions)) {
const joint = this.joints.get(jointName);
if (joint) {
// 应用平滑
if (smoothing > 0) {
position.lerp(joint.position, 1 - smoothing);
}
joint.position.copy(position);
}
}
// 更新骨骼方向和长度
this.updateBones();
}
mapKeypointsTo3D(keypoints) {
const positions = {};
const scale = 0.1; // 缩放因子
// 关键点映射表
const keypointMapping = {
'nose': 'head',
'left_shoulder': 'left_shoulder',
'right_shoulder': 'right_shoulder',
'left_elbow': 'left_elbow',
'right_elbow': 'right_elbow',
'left_wrist': 'left_wrist',
'right_wrist': 'right_wrist',
'left_hip': 'left_hip',
'right_hip': 'right_hip',
'left_knee': 'left_knee',
'right_knee': 'right_knee',
'left_ankle': 'left_ankle',
'right_ankle': 'right_ankle'
};
// 计算中心点（臀部）
const leftHip = keypoints.find(k => k.name === 'left_hip');
const rightHip = keypoints.find(k => k.name === 'right_hip');
if (leftHip && rightHip) {
const hipCenter = {
x: (leftHip.x + rightHip.x) / 2,
y: (leftHip.y + rightHip.y) / 2
};
// 设置臀部位置为原点
positions.hip = new THREE.Vector3(0, 0, 0);
// 计算其他关节的相对位置
for (const [keypointName, jointName] of Object.entries(keypointMapping)) {
const keypoint = keypoints.find(k => k.name === keypointName);
if (keypoint && keypoint.score >= 0.3) {
const x = (keypoint.x - hipCenter.x) * scale;
const y = -(keypoint.y - hipCenter.y) * scale; // Y轴翻转
const z = 0; // 初始深度为0
positions[jointName] = new THREE.Vector3(x, y, z);
}
}
// 计算肩部中心
const leftShoulder = positions.left_shoulder;
const rightShoulder = positions.right_shoulder;
if (leftShoulder && rightShoulder) {
positions.shoulder_center = new THREE.Vector3()
.addVectors(leftShoulder, rightShoulder)
.multiplyScalar(0.5);
}
}
return positions;
}
updateBones() {
for (const [boneName, bone] of this.bones) {
const startJoint = this.joints.get(bone.userData.startJoint);
const endJoint = this.joints.get(bone.userData.endJoint);
if (startJoint && endJoint) {
// 计算骨骼方向
const direction = new THREE.Vector3()
.subVectors(endJoint.position, startJoint.position);
const length = direction.length();
if (length > 0) {
// 设置骨骼位置（起点和终点的中点）
bone.position.copy(startJoint.position)
.add(endJoint.position)
.multiplyScalar(0.5);
// 设置骨骼方向
bone.lookAt(endJoint.position);
// 调整圆柱体方向（默认朝向Y轴）
bone.rotateX(Math.PI / 2);
// 设置骨骼长度
bone.scale.set(1, length, 1);
}
}
}
}
clearCharacter() {
if (this.character) {
this.scene.remove(this.character);
// 清理几何体和材质
this.bones.forEach(bone => {
bone.geometry.dispose();
bone.material.dispose();
});
this.joints.forEach(joint => {
joint.geometry.dispose();
joint.material.dispose();
});
this.bones.clear();
this.joints.clear();
this.character = null;
}
}
// 重置角色姿态
resetPose() {
this.joints.forEach(joint => {
joint.position.set(0, 0, 0);
});
this.updateBones();
}
}
// 姿态可视化器
class PoseVisualizer {
constructor(canvas) {
this.canvas = canvas;
this.ctx = canvas.getContext('2d');
this.keypointRadius = 4;
this.skeletonColor = '#00ff00';
this.keypointColor = '#ff0000';
}
drawPose(pose, videoWidth, videoHeight) {
if (!pose || !pose.keypoints) return;
// 清除画布
this.ctx.clearRect(0, 0, this.canvas.width, this.canvas.height);
// 设置画布尺寸匹配视频
this.canvas.width = videoWidth;
this.canvas.height = videoHeight;
// 绘制骨骼
this.drawSkeleton(pose.keypoints);
// 绘制关键点
this.drawKeypoints(pose.keypoints);
}
drawSkeleton(keypoints) {
const connections = [
// 身体主干
['left_shoulder', 'right_shoulder'],
['left_shoulder', 'left_hip'],
['right_shoulder', 'right_hip'],
['left_hip', 'right_hip'],
// 左臂
['left_shoulder', 'left_elbow'],
['left_elbow', 'left_wrist'],
// 右臂
['right_shoulder', 'right_elbow'],
['right_elbow', 'right_wrist'],
// 左腿
['left_hip', 'left_knee'],
['left_knee', 'left_ankle'],
// 右腿
['right_hip', 'right_knee'],
['right_knee', 'right_ankle'],
// 面部（简化）
['nose', 'left_eye'],
['nose', 'right_eye'],
['left_eye', 'left_ear'],
['right_eye', 'right_ear']
];
this.ctx.strokeStyle = this.skeletonColor;
this.ctx.lineWidth = 2;
connections.forEach(([start, end]) => {
const startPoint = keypoints.find(k => k.name === start);
const endPoint = keypoints.find(k => k.name === end);
if (startPoint && endPoint && startPoint.score > 0.3 && endPoint.score > 0.3) {
this.ctx.beginPath();
this.ctx.moveTo(startPoint.x, startPoint.y);
this.ctx.lineTo(endPoint.x, endPoint.y);
this.ctx.stroke();
}
});
}
drawKeypoints(keypoints) {
keypoints.forEach(keypoint => {
if (keypoint.score > 0.3) {
// 绘制关键点
this.ctx.fillStyle = this.keypointColor;
this.ctx.beginPath();
this.ctx.arc(keypoint.x, keypoint.y, this.keypointRadius, 0, 2 * Math.PI);
this.ctx.fill();
// 绘制置信度圆环
const ringRadius = this.keypointRadius * (1 + keypoint.score);
this.ctx.strokeStyle = `rgba(255, 255, 255, ${keypoint.score})`;
this.ctx.lineWidth = 1;
this.ctx.beginPath();
this.ctx.arc(keypoint.x, keypoint.y, ringRadius, 0, 2 * Math.PI);
this.ctx.stroke();
}
});
}
}
export default {
name: 'PoseEstimation',
setup() {
// 响应式状态
const videoElement = ref(null);
const poseCanvas = ref(null);
const characterCanvas = ref(null);
const isCameraActive = ref(false);
const isLoading = ref(false);
const loadingProgress = ref(0);
const selectedModel = ref('movenet');
const modelComplexity = ref('lite');
const minPoseConfidence = ref(0.3);
const minKeypointConfidence = ref(0.3);
const characterType = ref('stickman');
const boneThickness = ref(0.1);
const jointSize = ref(0.15);
const smoothingFactor = ref(0.5);
const showCharacter = ref(true);
const enableRecording = ref(false);
const isRecording = ref(false);
const enablePlayback = ref(false);
const isPlaying = ref(false);
const enable3DReconstruction = ref(false);
const enableSkeletonSmoothing = ref(true);
const showKeypoints = ref(true);
const showSkeleton = ref(true);
const mirrorMode = ref('horizontal');
const showKeypointPanel = ref(false);
// 性能统计
const detectionFPS = ref(0);
const renderFPS = ref(0);
const detectionTime = ref(0);
const keypointCount = ref(0);
const trackingQuality = ref('未知');
// 当前关键点数据
const currentKeypoints = ref([]);
// 计算属性
const trackingQualityClass = computed(() => {
const quality = trackingQuality.value;
if (quality === '优秀') return 'excellent';
if (quality === '良好') return 'good';
if (quality === '一般') return 'fair';
return 'poor';
});
// 管理器实例
let poseManager, characterManager, poseVisualizer;
let renderer, scene, camera;
let animationFrameId;
let detectionFrameCount = 0;
let renderFrameCount = 0;
let lastFpsUpdate = 0;
let videoStream = null;
// 初始化
const init = async () => {
isLoading.value = true;
try {
await initPoseEstimation();
await init3DRenderer();
initPoseVisualizer();
isLoading.value = false;
} catch (error) {
console.error('初始化失败:', error);
isLoading.value = false;
}
};
// 初始化姿态估计
const initPoseEstimation = async () => {
poseManager = new PoseEstimationManager();
loadingProgress.value = 50;
await poseManager.initialize(selectedModel.value, modelComplexity.value);
loadingProgress.value = 100;
};
// 初始化3D渲染器
const init3DRenderer = () => {
renderer = new THREE.WebGLRenderer({
canvas: characterCanvas.value,
antialias: true
});
renderer.setSize(characterCanvas.value.clientWidth, characterCanvas.value.clientHeight);
renderer.setClearColor(0x222222);
scene = new THREE.Scene();
camera = new THREE.PerspectiveCamera(75,
characterCanvas.value.clientWidth / characterCanvas.value.clientHeight,
0.1, 1000
);
camera.position.set(0, 0, 5);
camera.lookAt(0, 0, 0);
characterManager = new CharacterManager(renderer, scene, camera);
characterManager.createStickFigure();
};
// 初始化姿态可视化
const initPoseVisualizer = () => {
poseVisualizer = new PoseVisualizer(poseCanvas.value);
};
// 切换摄像头
const toggleCamera = async () => {
if (isCameraActive.value) {
stopCamera();
} else {
await startCamera();
}
};
// 启动摄像头
const startCamera = async () => {
try {
const constraints = {
video: {
width: { ideal: 640 },
height: { ideal: 480 },
facingMode: 'user'
}
};
videoStream = await navigator.mediaDevices.getUserMedia(constraints);
videoElement.value.srcObject = videoStream;
isCameraActive.value = true;
// 开始姿态估计循环
startPoseEstimationLoop();
} catch (error) {
console.error('摄像头访问失败:', error);
alert('无法访问摄像头，请检查权限设置');
}
};
// 停止摄像头
const stopCamera = () => {
if (videoStream) {
videoStream.getTracks().forEach(track => track.stop());
videoStream = null;
}
videoElement.value.srcObject = null;
isCameraActive.value = false;
if (animationFrameId) {
cancelAnimationFrame(animationFrameId);
}
};
// 视频准备就绪
const onVideoReady = () => {
// 设置画布尺寸匹配视频
if (poseCanvas.value) {
poseCanvas.value.width = videoElement.value.videoWidth;
poseCanvas.value.height = videoElement.value.videoHeight;
}
};
// 开始姿态估计循环
const startPoseEstimationLoop = () => {
const estimatePose = async () => {
if (!isCameraActive.value) return;
try {
// 估计姿态
const result = await poseManager.estimatePoses(
videoElement.value,
minPoseConfidence.value
);
if (result.poses.length > 0) {
const pose = result.poses[0];
// 更新关键点数据
currentKeypoints.value = pose.keypoints.map(kp => ({
name: kp.name,
score: kp.score,
position: { x: kp.x, y: kp.y }
}));
// 可视化姿态
if (showSkeleton.value || showKeypoints.value) {
poseVisualizer.drawPose(pose,
videoElement.value.videoWidth,
videoElement.value.videoHeight
);
}
// 更新3D角色
if (showCharacter.value) {
characterManager.updatePose(pose.keypoints, smoothingFactor.value);
}
// 更新性能统计
detectionTime.value = result.detectionTime.toFixed(1);
keypointCount.value = result.keypointCount;
detectionFrameCount++;
}
// 更新跟踪质量
updateTrackingQuality(result);
} catch (error) {
console.error('姿态估计错误:', error);
}
// 继续下一帧
animationFrameId = requestAnimationFrame(estimatePose);
};
estimatePose();
};
// 更新跟踪质量
const updateTrackingQuality = (result) => {
if (result.poses.length === 0) {
trackingQuality.value = '无检测';
} else {
const pose = result.poses[0];
const visibleKeypoints = pose.keypoints.filter(kp => kp.score > minKeypointConfidence.value).length;
const totalKeypoints = pose.keypoints.length;
const visibilityRatio = visibleKeypoints / totalKeypoints;
if (visibilityRatio > 0.8) trackingQuality.value = '优秀';
else if (visibilityRatio > 0.6) trackingQuality.value = '良好';
else if (visibilityRatio > 0.4) trackingQuality.value = '一般';
else trackingQuality.value = '较差';
}
};
// 切换镜像模式
const toggleMirror = () => {
mirrorMode.value = mirrorMode.value === 'horizontal' ? 'none' : 'horizontal';
// 实际实现需要更新可视化
};
// 截图
const takeSnapshot = () => {
const canvas = document.createElement('canvas');
const ctx = canvas.getContext('2d');
canvas.width = videoElement.value.videoWidth;
canvas.height = videoElement.value.videoHeight;
// 绘制视频帧
ctx.drawImage(videoElement.value, 0, 0);
// 绘制姿态覆盖
if (poseCanvas.value) {
ctx.drawImage(poseCanvas.value, 0, 0);
}
// 创建下载链接
const link = document.createElement('a');
link.download = `pose-snapshot-${Date.now()}.png`;
link.href = canvas.toDataURL();
link.click();
};
// 切换角色显示
const toggleCharacter = () => {
showCharacter.value = !showCharacter.value;
if (showCharacter.value && characterManager.character) {
characterManager.character.visible = true;
} else if (characterManager.character) {
characterManager.character.visible = false;
}
};
// 重置姿势
const resetPose = () => {
characterManager.resetPose();
};
// 校准姿态
const calibratePose = () => {
// 实现校准逻辑
alert('校准功能开发中...');
};
// 加载模型
const loadModel = async () => {
if (poseManager) {
poseManager.dispose();
}
await initPoseEstimation();
};
// 加载角色
const loadCharacter = () => {
if (characterManager) {
switch (characterType.value) {
case 'stickman':
characterManager.createStickFigure();
break;
case 'humanoid':
// 加载人形角色
break;
case 'robot':
// 加载机器人角色
break;
}
}
};
// 开始录制
const startRecording = () => {
isRecording.value = true;
// 实现录制逻辑
};
// 停止录制
const stopRecording = () => {
isRecording.value = false;
};
// 保存录制
const saveRecording = () => {
// 实现保存逻辑
alert('保存功能开发中...');
};
// 加载录制
const loadRecording = () => {
// 实现加载逻辑
alert('加载功能开发中...');
};
// 播放录制
const playRecording = () => {
isPlaying.value = true;
// 实现播放逻辑
};
// 暂停录制
const pauseRecording = () => {
isPlaying.value = false;
};
// 性能监控循环
const startPerformanceMonitor = () => {
const updateStats = () => {
const now = performance.now();
if (now - lastFpsUpdate >= 1000) {
detectionFPS.value = Math.round((detectionFrameCount * 1000) / (now - lastFpsUpdate));
renderFPS.value = Math.round((renderFrameCount * 1000) / (now - lastFpsUpdate));
detectionFrameCount = 0;
renderFrameCount = 0;
lastFpsUpdate = now;
}
requestAnimationFrame(updateStats);
};
updateStats();
};
// 3D渲染循环
const startRenderLoop = () => {
const render = () => {
if (showCharacter.value) {
renderer.render(scene, camera);
renderFrameCount++;
}
requestAnimationFrame(render);
};
render();
};
onMounted(() => {
init();
startPerformanceMonitor();
startRenderLoop();
window.addEventListener('resize', handleResize);
});
onUnmounted(() => {
stopCamera();
if (poseManager) {
poseManager.dispose();
}
if (animationFrameId) {
cancelAnimationFrame(animationFrameId);
}
window.removeEventListener('resize', handleResize);
});
const handleResize = () => {
if (renderer && camera && characterCanvas.value) {
camera.aspect = characterCanvas.value.clientWidth / characterCanvas.value.clientHeight;
camera.updateProjectionMatrix();
renderer.setSize(characterCanvas.value.clientWidth, characterCanvas.value.clientHeight);
}
};
return {
// 模板引用
videoElement,
poseCanvas,
characterCanvas,
// 状态数据
isCameraActive,
isLoading,
loadingProgress,
selectedModel,
modelComplexity,
minPoseConfidence,
minKeypointConfidence,
characterType,
boneThickness,
jointSize,
smoothingFactor,
showCharacter,
enableRecording,
isRecording,
enablePlayback,
isPlaying,
enable3DReconstruction,
enableSkeletonSmoothing,
showKeypoints,
showSkeleton,
mirrorMode,
showKeypointPanel,
detectionFPS,
renderFPS,
detectionTime,
keypointCount,
trackingQuality,
currentKeypoints,
// 计算属性
trackingQualityClass,
// 方法
toggleCamera,
onVideoReady,
toggleMirror,
takeSnapshot,
toggleCharacter,
resetPose,
calibratePose,
loadModel,
loadCharacter,
startRecording,
stopRecording,
saveRecording,
loadRecording,
playRecording,
pauseRecording
};
}
};
</script>
<style scoped>
.pose-estimation-container {
width: 100%;
height: 100vh;
display: flex;
background: #1a1a1a;
color: white;
overflow: hidden;
}
.main-view {
flex: 1;
display: flex;
padding: 20px;
gap: 20px;
}
.video-section, .character-section {
flex: 1;
display: flex;
flex-direction: column;
background: #2d2d2d;
border-radius: 8px;
padding: 15px;
position: relative;
}
.video-feed, .character-canvas {
width: 100%;
height: 400px;
background: #000;
border-radius: 4px;
object-fit: cover;
}
.pose-overlay {
position: absolute;
top: 15px;
left: 15px;
width: calc(100% - 30px);
height: 400px;
pointer-events: none;
}
.video-controls, .character-controls {
display: flex;
gap: 10px;
margin-top: 15px;
justify-content: center;
}
.control-button {
padding: 8px 16px;
background: #444;
color: white;
border: none;
border-radius: 4px;
cursor: pointer;
font-size: 12px;
transition: background 0.3s;
}
.control-button:hover:not(:disabled) {
background: #555;
}
.control-button:disabled {
opacity: 0.6;
cursor: not-allowed;
}
.control-panel {
width: 350px;
background: #2d2d2d;
padding: 20px;
overflow-y: auto;
border-left: 1px solid #444;
}
.panel-section {
margin-bottom: 25px;
padding-bottom: 20px;
border-bottom: 1px solid #444;
}
.panel-section:last-child {
margin-bottom: 0;
border-bottom: none;
}
.panel-section h3 {
color: #00ff88;
margin-bottom: 15px;
font-size: 16px;
}
.model-config, .character-config, .performance-monitor, .mocap-controls, .advanced-settings {
display: flex;
flex-direction: column;
gap: 15px;
}
.config-group, .setting-group, .control-group {
display: flex;
flex-direction: column;
gap: 8px;
}
.config-group label, .setting-group label {
color: #ccc;
font-size: 14px;
}
.config-group select, .config-group input[type="range"] {
padding: 8px 12px;
background: #444;
border: 1px solid #666;
border-radius: 4px;
color: white;
font-size: 14px;
}
.config-group span {
color: #00ff88;
font-size: 12px;
text-align: right;
}
.performance-item {
display: flex;
justify-content: space-between;
align-items: center;
padding: 8px 0;
border-bottom: 1px solid #444;
}
.performance-item:last-child {
border-bottom: none;
}
.performance-item span:first-child {
color: #ccc;
}
.performance-item span:last-child {
color: #00ff88;
font-weight: bold;
}
.performance-item .excellent { color: #48bb78; }
.performance-item .good { color: #4299e1; }
.performance-item .fair { color: #ed8936; }
.performance-item .poor { color: #f56565; }
.recording-controls, .playback-controls {
display: flex;
flex-direction: column;
gap: 8px;
margin-top: 10px;
}
.setting-group label {
display: flex;
align-items: center;
gap: 8px;
cursor: pointer;
}
.setting-group input[type="checkbox"] {
margin: 0;
}
.keypoint-panel {
position: absolute;
top: 20px;
right: 370px;
background: rgba(45, 45, 45, 0.95);
border-radius: 8px;
border: 1px solid #444;
backdrop-filter: blur(10px);
max-width: 300px;
max-height: 400px;
overflow: hidden;
}
.panel-header {
padding: 15px;
background: #1a1a1a;
display: flex;
justify-content: space-between;
align-items: center;
border-bottom: 1px solid #444;
}
.panel-header h4 {
margin: 0;
color: #00ff88;
}
.close-button {
background: none;
border: none;
color: #ccc;
font-size: 20px;
cursor: pointer;
padding: 0;
width: 24px;
height: 24px;
display: flex;
align-items: center;
justify-content: center;
}
.keypoint-list {
padding: 15px;
display: flex;
flex-direction: column;
gap: 6px;
max-height: 350px;
overflow-y: auto;
}
.keypoint-item {
display: flex;
justify-content: space-between;
align-items: center;
padding: 6px 8px;
background: #444;
border-radius: 4px;
font-size: 11px;
border-left: 3px solid #666;
}
.keypoint-item.active {
border-left-color: #00ff88;
}
.keypoint-name {
font-weight: bold;
min-width: 80px;
}
.keypoint-score {
color: #00ff88;
min-width: 40px;
text-align: right;
}
.keypoint-position {
color: #ccc;
font-family: monospace;
min-width: 80px;
text-align: right;
}
.loading-overlay {
position: fixed;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: rgba(0, 0, 0, 0.8);
display: flex;
justify-content: center;
align-items: center;
z-index: 1000;
}
.loading-content {
text-align: center;
color: white;
}
.spinner {
width: 50px;
height: 50px;
border: 4px solid rgba(255, 255, 255, 0.3);
border-top: 4px solid #00ff88;
border-radius: 50%;
animation: spin 1s linear infinite;
margin: 0 auto 20px;
}
.loading-content h3 {
margin: 0 0 10px 0;
}
.loading-content p {
margin: 0;
color: #00ff88;
font-size: 18px;
font-weight: bold;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
/* 响应式设计 */
@media (max-width: 1024px) {
.pose-estimation-container {
flex-direction: column;
}
.control-panel {
width: 100%;
height: 400px;
}
.main-view {
height: calc(100vh - 400px);
flex-direction: column;
}
.keypoint-panel {
right: 20px;
}
}
@media (max-width: 768px) {
.main-view {
padding: 10px;
}
.video-controls, .character-controls {
flex-wrap: wrap;
}
.keypoint-panel {
display: none;
}
}
</style>高级特性
3D姿态重建算法
javascript
// 3D姿态重建器
class Pose3DReconstructor {
constructor() {
this.cameraParams = {
focalLength: 1000,
principalPoint: { x: 0, y: 0 }
};
this.skeletonModel = this.createSkeletonModel();
}
// 从2D关键点重建3D姿态
reconstruct3DPose(keypoints2D) {
const keypoints3D = {};
// 使用骨骼长度约束和运动学先验
const rootPosition = this.estimateRootPosition(keypoints2D);
for (const [name, keypoint2D] of Object.entries(keypoints2D)) {
if (keypoint2D.score > 0.3) {
// 反投影到3D空间
const position3D = this.backprojectTo3D(keypoint2D, rootPosition);
keypoints3D[name] = {
position: position3D,
score: keypoint2D.score
};
}
}
// 应用运动学约束
this.applyKinematicConstraints(keypoints3D);
return keypoints3D;
}
estimateRootPosition(keypoints2D) {
// 使用臀部关键点作为根节点
const leftHip = keypoints2D.left_hip;
const rightHip = keypoints2D.right_hip;
if (leftHip && rightHip) {
const hipCenter2D = {
x: (leftHip.x + rightHip.x) / 2,
y: (leftHip.y + rightHip.y) / 2
};
// 假设初始深度为0
return this.backprojectTo3D(hipCenter2D, { x: 0, y: 0, z: 0 });
}
return { x: 0, y: 0, z: 0 };
}
backprojectTo3D(keypoint2D, referencePoint) {
// 简化反投影：使用针孔相机模型
const x = (keypoint2D.x - this.cameraParams.principalPoint.x) / this.cameraParams.focalLength;
const y = (keypoint2D.y - this.cameraParams.principalPoint.y) / this.cameraParams.focalLength;
// 使用参考点深度估计
const z = referencePoint.z || 0;
return { x, y, z };
}
applyKinematicConstraints(keypoints3D) {
// 应用骨骼长度约束
const boneLengths = this.estimateBoneLengths(keypoints3D);
this.enforceBoneLengths(keypoints3D, boneLengths);
// 应用关节角度限制
this.enforceJointLimits(keypoints3D);
}
estimateBoneLengths(keypoints3D) {
// 基于人体比例估计骨骼长度
const lengths = {};
const connections = [
['left_shoulder', 'left_elbow', 'upper_arm'],
['left_elbow', 'left_wrist', 'lower_arm'],
['left_hip', 'left_knee', 'upper_leg'],
['left_knee', 'left_ankle', 'lower_leg']
];
connections.forEach(([start, end, name]) => {
const startPoint = keypoints3D[start];
const endPoint = keypoints3D[end];
if (startPoint && endPoint) {
const length = this.calculateDistance(startPoint.position, endPoint.position);
lengths[name] = length;
}
});
return lengths;
}
calculateDistance(p1, p2) {
return Math.sqrt(
Math.pow(p2.x - p1.x, 2) +
Math.pow(p2.y - p1.y, 2) +
Math.pow(p2.z - p1.z, 2)
);
}
enforceBoneLengths(keypoints3D, targetLengths) {
// 迭代调整骨骼长度
for (const [boneName, targetLength] of Object.entries(targetLengths)) {
const [start, end] = this.getBoneJoints(boneName);
const startPoint = keypoints3D[start];
const endPoint = keypoints3D[end];
if (startPoint && endPoint) {
const currentLength = this.calculateDistance(startPoint.position, endPoint.position);
const scale = targetLength / currentLength;
if (Math.abs(scale - 1) > 0.1) {
// 调整骨骼长度
const direction = {
x: endPoint.position.x - startPoint.position.x,
y: endPoint.position.y - startPoint.position.y,
z: endPoint.position.z - startPoint.position.z
};
endPoint.position.x = startPoint.position.x + direction.x * scale;
endPoint.position.y = startPoint.position.y + direction.y * scale;
endPoint.position.z = startPoint.position.z + direction.z * scale;
}
}
}
}
}运动重定向系统
javascript
// 运动重定向器
class MotionRetargeter {
constructor(sourceSkeleton, targetSkeleton) {
this.sourceSkeleton = sourceSkeleton;
this.targetSkeleton = targetSkeleton;
this.boneMapping = this.createBoneMapping();
}
createBoneMapping() {
// 定义源骨架和目标骨架之间的骨骼映射
return {
'spine': 'spine',
'left_upper_arm': 'left_upper_arm',
'left_lower_arm': 'left_lower_arm',
'right_upper_arm': 'right_upper_arm',
'right_lower_arm': 'right_lower_arm',
'left_upper_leg': 'left_upper_leg',
'left_lower_leg': 'left_lower_leg',
'right_upper_leg': 'right_upper_leg',
'right_lower_leg': 'right_lower_leg'
};
}
retargetMotion(sourcePose) {
const targetPose = {};
for (const [sourceBone, targetBone] of Object.entries(this.boneMapping)) {
const sourceRotation = sourcePose[sourceBone];
if (sourceRotation) {
// 应用旋转到目标骨骼
targetPose[targetBone] = this.adjustRotationForSkeleton(
sourceRotation,
sourceBone,
targetBone
);
}
}
return targetPose;
}
adjustRotationForSkeleton(rotation, sourceBone, targetBone) {
// 根据骨架差异调整旋转
const adjustment = this.calculateBoneAdjustment(sourceBone, targetBone);
// 应用调整
const adjustedRotation = new THREE.Quaternion();
adjustedRotation.multiplyQuaternions(rotation, adjustment);
return adjustedRotation;
}
calculateBoneAdjustment(sourceBone, targetBone) {
// 计算源骨骼和目标骨骼之间的方向差异
// 简化实现
return new THREE.Quaternion();
}
}本节展示了如何将TensorFlow.js的机器学习能力与Three.js的3D渲染能力结合，实现实时的姿态估计和角色动画驱动。这种技术为虚拟试衣、运动分析、游戏交互等应用提供了强大的基础。