双相机结构光成像与图像交叉融合实现
下面我将详细介绍如何使用Python实现双相机结构光成像系统及其图像交叉融合技术。这个系统通常用于三维重建、工业检测等领域。
系统架构概述
双相机结构光系统通常包含以下组件：
- 两个同步的工业相机
- 结构光投影仪（DLP或LCD）
- 计算机处理系统
- 标定装置
实现步骤
1. 硬件设置与相机同步
python
import cv2
import numpy as np
import time
class DualCameraSystem:
def __init__(self, cam1_index=0, cam2_index=1):
"""初始化双相机系统"""
self.cam1 = cv2.VideoCapture(cam1_index)
self.cam2 = cv2.VideoCapture(cam2_index)
# 设置相机参数
self._setup_cameras()
def _setup_cameras(self):
"""配置相机参数"""
# 设置分辨率
self.cam1.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
self.cam1.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
self.cam2.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
self.cam2.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
# 设置帧率
self.cam1.set(cv2.CAP_PROP_FPS, 30)
self.cam2.set(cv2.CAP_PROP_FPS, 30)
# 设置曝光时间（根据实际相机支持）
self.cam1.set(cv2.CAP_PROP_EXPOSURE, -4) # 示例值
self.cam2.set(cv2.CAP_PROP_EXPOSURE, -4)
def capture_synchronized(self):
"""同步捕获图像"""
# 实际项目中可能需要硬件触发实现精确同步
ret1, frame1 = self.cam1.read()
ret2, frame2 = self.cam2.read()
if ret1 and ret2:
return frame1, frame2
else:
raise RuntimeError("相机捕获失败")
def release(self):
"""释放相机资源"""
self.cam1.release()
self.cam2.release()2. 结构光图案生成
python
class StructuredLightPatterns:
@staticmethod
def generate_gray_code_patterns(width, height, bits=8):
"""生成格雷码图案序列"""
patterns = []
# 水平方向格雷码
for i in range(bits):
pattern = np.zeros((height, width), dtype=np.uint8)
period = 2 ** (bits - i)
value = 0
for x in range(width):
if x % (period // 2) == 0:
value = 255 - value
pattern[:, x] = value
patterns.append(pattern)
# 垂直方向格雷码（如果需要）
for i in range(bits):
pattern = np.zeros((height, width), dtype=np.uint8)
period = 2 ** (bits - i)
value = 0
for y in range(height):
if y % (period // 2) == 0:
value = 255 - value
pattern[y, :] = value
patterns.append(pattern)
return patterns
@staticmethod
def generate_sinusoidal_patterns(width, height, frequencies=[10, 20, 40], phases=4):
"""生成正弦相位偏移图案"""
patterns = []
x = np.arange(width)
y = np.arange(height)
xx, yy = np.meshgrid(x, y)
for freq in frequencies:
for phase in range(phases):
phi = 2 * np.pi * phase / phases
pattern = 127.5 + 127.5 * np.sin(2 * np.pi * freq * xx / width + phi)
patterns.append(pattern.astype(np.uint8))
return patterns3. 相机标定与极线校正
python
class StereoCalibration:
def __init__(self):
self.camera_matrix1 = None
self.dist_coeffs1 = None
self.camera_matrix2 = None
self.dist_coeffs2 = None
self.R = None # 旋转矩阵
self.T = None # 平移向量
self.E = None # 本质矩阵
self.F = None # 基础矩阵
def calibrate(self, image_points1, image_points2, board_size, square_size):
"""双目标定"""
# 生成物体点 (0,0,0), (1,0,0), (2,0,0) ..., (board_size[0]-1, board_size[1]-1,0)
objp = np.zeros((board_size[0] * board_size[1], 3), np.float32)
objp[:, :2] = np.mgrid[0:board_size[0], 0:board_size[1]].T.reshape(-1, 2)
objp *= square_size
obj_points = [objp] * len(image_points1)
# 单目标定
ret1, mtx1, dist1, rvecs1, tvecs1 = cv2.calibrateCamera(
obj_points, image_points1, image_points1[0].shape[::-1], None, None)
ret2, mtx2, dist2, rvecs2, tvecs2 = cv2.calibrateCamera(
obj_points, image_points2, image_points2[0].shape[::-1], None, None)
# 双目标定
flags = cv2.CALIB_FIX_INTRINSIC
ret, M1, d1, M2, d2, R, T, E, F = cv2.stereoCalibrate(
obj_points, image_points1, image_points2,
mtx1, dist1, mtx2, dist2, image_points1[0].shape[::-1],
flags=flags)
self.camera_matrix1 = M1
self.dist_coeffs1 = d1
self.camera_matrix2 = M2
self.dist_coeffs2 = d2
self.R = R
self.T = T
self.E = E
self.F = F
return ret
def rectify(self, image_size):
"""计算校正映射"""
R1, R2, P1, P2, Q, roi1, roi2 = cv2.stereoRectify(
self.camera_matrix1, self.dist_coeffs1,
self.camera_matrix2, self.dist_coeffs2,
image_size, self.R, self.T)
# 计算校正映射
map1x, map1y = cv2.initUndistortRectifyMap(
self.camera_matrix1, self.dist_coeffs1, R1, P1,
image_size, cv2.CV_32FC1)
map2x, map2y = cv2.initUndistortRectifyMap(
self.camera_matrix2, self.dist_coeffs2, R2, P2,
image_size, cv2.CV_32FC1)
return (map1x, map1y), (map2x, map2y), Q4. 相位解包裹与深度计算
python
class PhaseProcessing:
@staticmethod
def compute_phase(images, phases=4):
"""从相位偏移图像计算相位"""
if len(images) != phases:
raise ValueError(f"需要{phases}幅图像，但得到了{len(images)}")
numerator = 0.0
denominator = 0.0
for k, img in enumerate(images):
img_float = img.astype(np.float32)
numerator += img_float * np.sin(2 * np.pi * k / phases)
denominator += img_float * np.cos(2 * np.pi * k / phases)
phase = np.arctan2(numerator, denominator)
return phase
@staticmethod
def unwrap_phase(gray_code_patterns, phase):
"""使用格雷码解包裹相位"""
# 将格雷码转换为二进制码
code = np.zeros_like(phase, dtype=np.int32)
for i, pattern in enumerate(gray_code_patterns):
code += (pattern > 127) * (2 ** i)
# 解包裹相位
unwrapped_phase = phase + 2 * np.pi * code
return unwrapped_phase
@staticmethod
def compute_depth(phase1, phase2, Q, baseline, wavelength):
"""从双相机相位计算深度"""
# 计算相位差
phase_diff = phase1 - phase2
# 避免相位跳变
phase_diff = np.where(phase_diff < -np.pi, phase_diff + 2*np.pi, phase_diff)
phase_diff = np.where(phase_diff > np.pi, phase_diff - 2*np.pi, phase_diff)
# 计算深度
depth = (wavelength * baseline) / (2 * np.pi) * phase_diff
return depth5. 图像交叉融合算法
python
class ImageFusion:
@staticmethod
def pyramid_fusion(img1, img2, levels=5):
"""基于金字塔的图像融合"""
# 生成高斯金字塔
gauss_pyr1 = [img1.astype(np.float32)]
gauss_pyr2 = [img2.astype(np.float32)]
for i in range(levels):
img1 = cv2.pyrDown(gauss_pyr1[-1])
img2 = cv2.pyrDown(gauss_pyr2[-1])
gauss_pyr1.append(img1)
gauss_pyr2.append(img2)
# 生成拉普拉斯金字塔
laplacian_pyr1 = [gauss_pyr1[levels-1]]
laplacian_pyr2 = [gauss_pyr2[levels-1]]
for i in range(levels-1, 0, -1):
expanded1 = cv2.pyrUp(gauss_pyr1[i])
expanded2 = cv2.pyrUp(gauss_pyr2[i])
l1 = gauss_pyr1[i-1] - expanded1
l2 = gauss_pyr2[i-1] - expanded2
laplacian_pyr1.append(l1)
laplacian_pyr2.append(l2)
# 融合拉普拉斯金字塔
fused_pyr = []
for l1, l2 in zip(laplacian_pyr1, laplacian_pyr2):
fused = (l1 + l2) / 2
fused_pyr.append(fused)
# 重建融合图像
fused_img = fused_pyr[0]
for i in range(1, levels):
fused_img = cv2.pyrUp(fused_img)
fused_img = fused_img + fused_pyr[i]
return np.clip(fused_img, 0, 255).astype(np.uint8)
@staticmethod
def wavelet_fusion(img1, img2, wavelet='db1', level=3):
"""基于小波的图像融合"""
import pywt
# 将图像转换为浮点型
img1 = img1.astype(np.float32)
img2 = img2.astype(np.float32)
# 多通道处理
if len(img1.shape) == 3:
fused_channels = []
for c in range(img1.shape[2]):
coeffs1 = pywt.wavedec2(img1[:,:,c], wavelet, level=level)
coeffs2 = pywt.wavedec2(img2[:,:,c], wavelet, level=level)
# 融合系数 - 这里使用简单的平均策略
fused_coeffs = []
for (c1, c2) in zip(coeffs1, coeffs2):
if isinstance(c1, tuple): # 细节系数
fused = tuple((a+b)/2 for a, b in zip(c1, c2))
else: # 近似系数
fused = (c1 + c2) / 2
fused_coeffs.append(fused)
# 重建图像
fused_channel = pywt.waverec2(fused_coeffs, wavelet)
fused_channels.append(fused_channel)
# 合并通道
fused_img = np.stack(fused_channels, axis=-1)
else:
# 单通道图像
coeffs1 = pywt.wavedec2(img1, wavelet, level=level)
coeffs2 = pywt.wavedec2(img2, wavelet, level=level)
# 融合系数
fused_coeffs = []
for (c1, c2) in zip(coeffs1, coeffs2):
if isinstance(c1, tuple):
fused = tuple((a+b)/2 for a, b in zip(c1, c2))
else:
fused = (c1 + c2) / 2
fused_coeffs.append(fused)
fused_img = pywt.waverec2(fused_coeffs, wavelet)
# 归一化并转换为uint8
fused_img = np.clip(fused_img, 0, 255)
return fused_img.astype(np.uint8)6. 完整系统集成
python
class StructuredLight3DScanner:
def __init__(self, cam1_index=0, cam2_index=1):
self.camera_system = DualCameraSystem(cam1_index, cam2_index)
self.calibration = StereoCalibration()
self.is_calibrated = False
self.rectify_maps = None
self.Q = None
def calibrate_system(self, checkerboard_size, square_size, num_images=15):
"""使用棋盘格标定系统"""
obj_points = [] # 3D点
img_points1 = [] # 相机1的2D点
img_points2 = [] # 相机2的2D点
pattern_found = 0
while pattern_found < num_images:
frame1, frame2 = self.camera_system.capture_synchronized()
gray1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
gray2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
ret1, corners1 = cv2.findChessboardCorners(
gray1, checkerboard_size, None)
ret2, corners2 = cv2.findChessboardCorners(
gray2, checkerboard_size, None)
if ret1 and ret2:
# 提高角点检测精度
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
cv2.cornerSubPix(gray1, corners1, (11,11), (-1,-1), criteria)
cv2.cornerSubPix(gray2, corners2, (11,11), (-1,-1), criteria)
img_points1.append(corners1)
img_points2.append(corners2)
pattern_found += 1
# 可视化
cv2.drawChessboardCorners(frame1, checkerboard_size, corners1, ret1)
cv2.drawChessboardCorners(frame2, checkerboard_size, corners2, ret2)
cv2.imshow('Camera 1', frame1)
cv2.imshow('Camera 2', frame2)
cv2.waitKey(500)
cv2.destroyAllWindows()
# 执行标定
image_size = gray1.shape[::-1]
self.calibration.calibrate(img_points1, img_points2,
checkerboard_size, square_size)
# 计算校正映射
self.rectify_maps, _, self.Q = self.calibration.rectify(image_size)
self.is_calibrated = True
def capture_patterns(self, patterns):
"""捕获结构光图案序列"""
all_images1 = []
all_images2 = []
for pattern in patterns:
# 在实际系统中，这里应该控制投影仪显示pattern
print(f"投影图案: {pattern.shape}")
# 等待投影稳定
time.sleep(0.5)
# 捕获图像
frame1, frame2 = self.camera_system.capture_synchronized()
gray1 = cv2.cvtColor(frame1, cv2.COLOR_BGR2GRAY)
gray2 = cv2.cvtColor(frame2, cv2.COLOR_BGR2GRAY)
all_images1.append(gray1)
all_images2.append(gray2)
return all_images1, all_images2
def reconstruct_3d(self, images1, images2, wavelength=40):
"""三维重建"""
if not self.is_calibrated:
raise RuntimeError("系统未标定")
# 校正图像
rectified1 = []
rectified2 = []
map1x, map1y = self.rectify_maps[0]
map2x, map2y = self.rectify_maps[1]
for img1, img2 in zip(images1, images2):
r1 = cv2.remap(img1, map1x, map1y, cv2.INTER_LINEAR)
r2 = cv2.remap(img2, map2x, map2y, cv2.INTER_LINEAR)
rectified1.append(r1)
rectified2.append(r2)
# 计算相位
phase_processor = PhaseProcessing()
# 假设前4幅是相位偏移图案
phase1 = phase_processor.compute_phase(rectified1[:4])
phase2 = phase_processor.compute_phase(rectified2[:4])
# 假设后8幅是格雷码图案（水平+垂直）
gray_code_patterns1 = rectified1[4:12]
gray_code_patterns2 = rectified2[4:12]
# 解包裹相位
unwrapped_phase1 = phase_processor.unwrap_phase(gray_code_patterns1[:4], phase1)
unwrapped_phase2 = phase_processor.unwrap_phase(gray_code_patterns2[:4], phase2)
# 计算深度
baseline = np.linalg.norm(self.calibration.T) # 基线距离
depth = phase_processor.compute_depth(
unwrapped_phase1, unwrapped_phase2, self.Q, baseline, wavelength)
return depth
def fuse_images(self, images1, images2, method='pyramid'):
"""融合双相机图像"""
fused_images = []
for img1, img2 in zip(images1, images2):
if method == 'pyramid':
fused = ImageFusion.pyramid_fusion(img1, img2)
elif method == 'wavelet':
fused = ImageFusion.wavelet_fusion(img1, img2)
else:
raise ValueError("不支持的融合方法")
fused_images.append(fused)
return fused_images
def close(self):
"""关闭系统"""
self.camera_system.release()使用示例
python
if __name__ == "__main__":
# 初始化扫描仪
scanner = StructuredLight3DScanner()
try:
# 标定系统
print("开始标定...")
scanner.calibrate_system((7,9), 0.025) # 7x9棋盘格，每个方格25mm
print("标定完成")
# 生成结构光图案
pattern_gen = StructuredLightPatterns()
patterns = []
patterns.extend(pattern_gen.generate_sinusoidal_patterns(1280, 720))
patterns.extend(pattern_gen.generate_gray_code_patterns(1280, 720, bits=4))
# 捕获图案
print("捕获结构光图案...")
images1, images2 = scanner.capture_patterns(patterns)
# 图像融合
print("图像融合...")
fused_images = scanner.fuse_images(images1, images2, method='pyramid')
# 三维重建
print("三维重建...")
depth_map = scanner.reconstruct_3d(images1, images2)
# 可视化结果
cv2.imshow("融合图像", fused_images[0])
cv2.imshow("深度图", cv2.normalize(depth_map, None, 0, 255, cv2.NORM_MINMAX).astype(np.uint8))
cv2.waitKey(0)
finally:
scanner.close()关键技术说明
-
结构光编码：
- 使用格雷码和正弦相位偏移图案进行编码
- 格雷码提供粗定位，正弦图案提供精确定位
-
相位解包裹：
- 结合格雷码和相位偏移技术解决相位模糊问题
- 实现高精度的相位测量
-
立体匹配：
- 利用双相机系统获取不同视角的相位信息
- 通过相位差计算三维坐标
-
图像融合：
- 金字塔融合：保留不同频率的信息
- 小波融合：在频域实现图像融合
-
系统标定：
- 精确的相机内参和外参标定
- 极线校正简化匹配过程
实际应用中的注意事项
-
硬件同步：
- 使用硬件触发器确保相机和投影仪的精确同步
- 考虑曝光时间和投影时序
-
环境光干扰：
- 在暗室环境中工作或使用窄带滤光片
- 考虑环境光补偿算法
-
标定精度：
- 使用高精度标定板
- 多次标定取平均值提高精度
-
实时性优化：
- 使用GPU加速计算密集型操作
- 优化算法实现实时处理
这个实现提供了双相机结构光系统的完整框架，包括图像采集、系统标定、结构光解码、三维重建和图像融合等功能。根据具体应用需求，可以进一步优化和扩展各个模块。