是否会出现blender里面的obj投影到原始视频流上存在偏移的情况，前几次关于blender的博客中也存在这种问题，可以去看看，接下来我们给出一张样图， 如下图所示：
【解决方法】：用重投影方式，投影矩阵去给原始图像增加一定偏移量，这个偏移量的产生是因为blender中obj物体旋转的角度偏移量导致的。
python
def process_video_with_objects(camera, video_path, output_dir, output_video_path, obj_dir, fps=30, camera_dict={}):
os.makedirs(output_dir, exist_ok=True)
K, R, T = camera_dict['K'], camera_dict['R'], camera_dict['T']
video = cv2.VideoCapture(video_path)
if not video.isOpened():
raise ValueError(f"Could not open video file: {video_path}")
input_width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
input_height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
out_video = cv2.VideoWriter(output_video_path, fourcc, fps, (input_width, input_height), True)
frame_idx = 0
while True:
ret, frame = video.read()
if not ret:
break
bpy.context.scene.frame_set(frame_idx)
scene = bpy.context.scene
# Load corresponding .obj file for the current frame
obj_files = sorted([f for f in os.listdir(obj_dir) if f.endswith('_{}.obj'.format(frame_idx + 1))])
clear_scene(visual_axis, visual_mark, base_keypoints_dict) # 清空非相机对象并筛选是否可视化坐标轴
for obj_file in obj_files:
bpy.ops.import_scene.obj(filepath=os.path.join(obj_dir, obj_file)) # 导入 obj 文件
# 计算世界坐标原点在相机局部坐标系中的位置
world_origin = Vector((0, 0, 0)) # 世界坐标原点
render = bpy.context.scene.render
# Get the two components to calculate M
modelview_matrix = camera.matrix_world.inverted()
# Get the depsgraph for the current scene
depsgraph = bpy.context.evaluated_depsgraph_get()
# Get the projection matrix
projection_matrix = camera.calc_matrix_camera(
depsgraph,
x=render.resolution_x,
y=render.resolution_y,
scale_x=render.pixel_aspect_x,
scale_y=render.pixel_aspect_y,
)
# Compute P' = M * P
p1 = np.dot(projection_matrix @ modelview_matrix, Vector((world_origin.x, world_origin.y, world_origin.z, 1)))
p2 = Vector(((p1[0] / p1[3]), (p1[1] / p1[3])))
pixel_x_now, pixel_y_now = (p2.x + 1) * 0.5 * render.resolution_x, (1 - p2.y) * 0.5 * render.resolution_y
pixel_target = project_points_3d_to_2d(np.array([[0, 0, 0]]), K, R, T)
shift_x_pixels, shift_y_pixels = (int(pixel_x_now - pixel_target[0][0])), (
int(pixel_y_now - pixel_target[0][1]))
frame = shift_image(frame, shift_x_pixels, shift_y_pixels)
scene.render.resolution_x = input_width
scene.render.resolution_y = input_height
scene.render.film_transparent = True
scene.camera = camera
output_frame_path = os.path.join(output_dir, f'rendered_{frame_idx:04d}.png')
scene.render.filepath = output_frame_path
bpy.context.view_layer.update()
bpy.ops.render.render(write_still=True)
for obj in bpy.context.selected_objects:
bpy.data.objects.remove(obj, do_unlink=True)
rendered_frame = cv2.imread(output_frame_path, cv2.IMREAD_UNCHANGED)
if rendered_frame is None:
continue
if rendered_frame.shape[2] == 3:
rendered_frame = cv2.cvtColor(rendered_frame, cv2.COLOR_BGR2BGRA)
alpha = rendered_frame[:, :, 3] / 255.0
for c in range(0, 3):
frame[:, :, c] = (1 - alpha) * frame[:, :, c] + alpha * rendered_frame[:, :, c]
out_video.write(frame)
frame_idx += 1
video.release()
out_video.release()
print(f"Processed video saved to {output_video_path}")结果可视化：
blender渲染里面设置透明度
python
def createMaterial(color=(0.8, 0.8, 0.8, 0.5), metallic=0, roughness=0., alpha=0.5):
color_count = [m.name for m in bpy.data.materials].count(str(color))
if color_count > 0:
color_index = [m.name for m in bpy.data.materials].index(str(color))
matg = bpy.data.materials[color_index]
else:
matg = bpy.data.materials.new(str(color))
matg.use_nodes = True
tree = matg.node_tree
nodes = tree.nodes
links = tree.links
for node in nodes:
nodes.remove(node)
# 添加必要的节点
output = nodes.new(type='ShaderNodeOutputMaterial')
bsdf = nodes.new(type='ShaderNodeBsdfPrincipled')
transparent = nodes.new(type='ShaderNodeBsdfTransparent')
mix_shader = nodes.new(type='ShaderNodeMixShader')
bsdf.inputs["Base Color"].default_value = color
bsdf.inputs["Metallic"].default_value = metallic
bsdf.inputs["Roughness"].default_value = roughness
mix_shader.inputs[0].default_value = 1 - alpha # 透明度控制
links.new(bsdf.outputs[0], mix_shader.inputs[2])
links.new(transparent.outputs[0], mix_shader.inputs[1])
links.new(mix_shader.outputs[0], output.inputs[0])
matg.blend_method = 'BLEND'
matg.shadow_method = 'HASHED'
return matg