1. 前期准备
- RGB相机：森云智能SG2-IMX390，1个
- 红外相机：艾睿光电IR-Pilot 640X-32G，1个
- 红外标定板：https://item.taobao.com/item.htm?_u=jp3fdd12b99&id=644506141871&spm=a1z09.2.0.0.5f822e8dKrxxYI
2.操作步骤
2.1 采集标定数据
两种模态相机均未进行内参标定，如果发现原始图片畸变较大，可以先进行内参标定。数据采集代码如下，加热红外标定板后断电，移动标定板到合适的位置，按下s键，同时保存IR图和RG图
python
#!/usr/bin/env python3
import cv2 , time
import numpy as np
ir_dev = "/dev/video6"
rgb_dev = "/dev/video0"
# define a video capture object
ir_vid = cv2.VideoCapture(ir_dev)
rgb_vid = cv2.VideoCapture(rgb_dev)
count = 0
while(True):
# Capture the video frame by frame
st_time = time.time()
ret, ir_frame = ir_vid.read()
# print(f"{time.time() - st_time}")
ret, rgb_frame = rgb_vid.read()
print(f"{time.time() - st_time}")
# Display the resulting frame
height, width = ir_frame.shape[:2]
#(512,1280)
index = [2*i+1 for i in range(width//2)]
vis_ir_frame = ir_frame[:,index,:]
vis_rgb_frame = cv2.resize(rgb_frame, (640,512))
cv2.imshow('IR frame', vis_ir_frame)
cv2.imshow('RGB frame', vis_rgb_frame)
key = cv2.waitKey(1) & 0xFF
if key == ord('q'):
break
if key == ord('s'):
cv2.imwrite(f"IR_{count}.png", vis_ir_frame)
cv2.imwrite(f"RGB_{count}.png", vis_rgb_frame)
count += 1
# After the loop release the cap object
ir_vid.release()
rgb_vid.release()
# Destroy all the windows
cv2.destroyAllWindows() 2.2 进行标定
核心操作是调用opencv函数cv2.findHomography计算两个相机之间的单应性矩阵，代码如下
python
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import cv2
import numpy as np
def find_chessboard(filename, pattern=(9,8), wind_name="rgb"):
# read input image
img = cv2.imread(filename)
# cv2.imshow("raw", img)
# img = cv2.undistort(img, camera_matrix, distortion_coefficients)
# convert the input image to a grayscale
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# Find the chess board corners
ret, corners = cv2.findChessboardCorners(gray, pattern, None)
# if chessboard corners are detected
if ret == True:
# Draw and display the corners
img = cv2.drawChessboardCorners(img, pattern, corners, ret)
#Draw number，打印角点编号，便于确定对应点
corners = np.ceil(corners[:,0,:])
for i, pt in enumerate(corners):
cv2.putText(img, str(i), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)
cv2.imshow(wind_name,img)
return corners
return None
if __name__ == '__main__' :
idx = 2 #0~71
rgb_img = cv2.imread(f"RGB_{idx}.png")
t_img = cv2.imread(f"IR_{idx}.png")
#chessboard grid nums in rgb ,注意观察，同一块标定板在RGB相机和红外相机中的格子说可能不一样
rgb_width, rgb_height = 9, 8
rgb_corners = find_chessboard(f"RGB_{idx}.png", (rgb_width, rgb_height), "rgb")
#chessboard grid nums in thermal
thermal_width, thermal_height = 11, 8
t_corners = find_chessboard(f"IR_{idx}.png", (thermal_width, thermal_height), "thermal")
if rgb_corners is not None and t_corners is not None:
# test the id correspondence between rgb and thermal corners
rgb_idx = 27 #可视化一个点，确认取对应点的过程是否正确
row, col = rgb_idx//rgb_width, rgb_idx%rgb_width
t_idx = row*thermal_width + col + 1
pt = rgb_corners[rgb_idx]
cv2.putText(rgb_img, str(rgb_idx), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)
pt = t_corners[t_idx]
cv2.putText(t_img, str(t_idx), (int(pt[0]),int(pt[1])), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0,255,0), 1)
cv2.imshow(f"Point {rgb_idx} on rgb", rgb_img)
cv2.imshow(f"Point {t_idx} on thermal", t_img)
# Calculate Homography
src_pts = []
for rgb_idx in range(len(rgb_corners)):
row, col = rgb_idx//9, rgb_idx%9
t_idx = row*11+col+1
src_pts.append(t_corners[t_idx])
h, status = cv2.findHomography(np.array(src_pts)[:,None,:], rgb_corners[:,None,:])
np.savetxt("calib.param", h)
# Warp source image to destination based on homography
t_warp = cv2.warpPerspective(t_img, h, (640,512), borderValue=(255,255,255))
#colorize
t_warp = cv2.applyColorMap(t_warp, cv2.COLORMAP_JET)
#mix rgb and thermal
alpha = 0.5
merge = cv2.addWeighted(rgb_img, alpha, t_warp, 1-alpha, gamma=0)
cv2.imshow("warp", merge)
cv2.waitKey(0)
cv2.destroyAllWindows()运行结果如下，观察红外和RGB图中角点的对应关系，编号已经可视化出来了
同时，也单独画出了1个对应后的点，如下图，可检查映射关系是否找对
最后，融合结果如下图所示：
