解决blender投影歪斜问题

是否会出现blender里面的obj投影到原始视频流上存在偏移的情况，前几次关于blender的博客中也存在这种问题，可以去看看，接下来我们给出一张样图，如下图所示：

【解决方法】：用重投影方式，投影矩阵去给原始图像增加一定偏移量，这个偏移量的产生是因为blender中obj物体旋转的角度偏移量导致的。

python 复制代码

def process_video_with_objects(camera, video_path, output_dir, output_video_path, obj_dir, fps=30, camera_dict={}):
    os.makedirs(output_dir, exist_ok=True)
    K, R, T = camera_dict['K'], camera_dict['R'], camera_dict['T']
    video = cv2.VideoCapture(video_path)
    if not video.isOpened():
        raise ValueError(f"Could not open video file: {video_path}")
    input_width = int(video.get(cv2.CAP_PROP_FRAME_WIDTH))
    input_height = int(video.get(cv2.CAP_PROP_FRAME_HEIGHT))
    fourcc = cv2.VideoWriter_fourcc(*'mp4v')
    out_video = cv2.VideoWriter(output_video_path, fourcc, fps, (input_width, input_height), True)
    frame_idx = 0
    while True:
        ret, frame = video.read()
        if not ret:
            break
        bpy.context.scene.frame_set(frame_idx)
        scene = bpy.context.scene
        # Load corresponding .obj file for the current frame
        obj_files = sorted([f for f in os.listdir(obj_dir) if f.endswith('_{}.obj'.format(frame_idx + 1))])
        clear_scene(visual_axis, visual_mark, base_keypoints_dict)  # 清空非相机对象并筛选是否可视化坐标轴
        for obj_file in obj_files:
            bpy.ops.import_scene.obj(filepath=os.path.join(obj_dir, obj_file))  # 导入 obj 文件
        # 计算世界坐标原点在相机局部坐标系中的位置
        world_origin = Vector((0, 0, 0))  # 世界坐标原点
        render = bpy.context.scene.render
        # Get the two components to calculate M
        modelview_matrix = camera.matrix_world.inverted()
        # Get the depsgraph for the current scene
        depsgraph = bpy.context.evaluated_depsgraph_get()
        # Get the projection matrix
        projection_matrix = camera.calc_matrix_camera(
                depsgraph,
                x=render.resolution_x,
                y=render.resolution_y,
                scale_x=render.pixel_aspect_x,
                scale_y=render.pixel_aspect_y,
        )
        # Compute P' = M * P
        p1 = np.dot(projection_matrix @ modelview_matrix,  Vector((world_origin.x, world_origin.y, world_origin.z, 1)))
        p2 = Vector(((p1[0] / p1[3]), (p1[1] / p1[3])))
        pixel_x_now, pixel_y_now = (p2.x + 1) * 0.5 * render.resolution_x, (1 - p2.y) * 0.5 * render.resolution_y
        pixel_target = project_points_3d_to_2d(np.array([[0, 0, 0]]), K, R, T)
        shift_x_pixels, shift_y_pixels = (int(pixel_x_now - pixel_target[0][0])), (
            int(pixel_y_now - pixel_target[0][1]))
        frame = shift_image(frame, shift_x_pixels, shift_y_pixels)
     
        scene.render.resolution_x = input_width
        scene.render.resolution_y = input_height
        scene.render.film_transparent = True
        scene.camera = camera

        output_frame_path = os.path.join(output_dir, f'rendered_{frame_idx:04d}.png')
        scene.render.filepath = output_frame_path
        bpy.context.view_layer.update()
        bpy.ops.render.render(write_still=True)
        for obj in bpy.context.selected_objects:
            bpy.data.objects.remove(obj, do_unlink=True)
        rendered_frame = cv2.imread(output_frame_path, cv2.IMREAD_UNCHANGED)
        if rendered_frame is None:
            continue

        if rendered_frame.shape[2] == 3:
            rendered_frame = cv2.cvtColor(rendered_frame, cv2.COLOR_BGR2BGRA)

        alpha = rendered_frame[:, :, 3] / 255.0
        for c in range(0, 3):
            frame[:, :, c] = (1 - alpha) * frame[:, :, c] + alpha * rendered_frame[:, :, c]

        out_video.write(frame)
        frame_idx += 1

    video.release()
    out_video.release()
    print(f"Processed video saved to {output_video_path}")

结果可视化：

blender渲染里面设置透明度

python 复制代码

def createMaterial(color=(0.8, 0.8, 0.8, 0.5), metallic=0, roughness=0., alpha=0.5):
    color_count = [m.name for m in bpy.data.materials].count(str(color))
    if color_count > 0:
        color_index = [m.name for m in bpy.data.materials].index(str(color))
        matg = bpy.data.materials[color_index]
    else:
        matg = bpy.data.materials.new(str(color))
        matg.use_nodes = True
        tree = matg.node_tree
        nodes = tree.nodes
        links = tree.links

        for node in nodes:
            nodes.remove(node)

        # 添加必要的节点
        output = nodes.new(type='ShaderNodeOutputMaterial')
        bsdf = nodes.new(type='ShaderNodeBsdfPrincipled')
        transparent = nodes.new(type='ShaderNodeBsdfTransparent')
        mix_shader = nodes.new(type='ShaderNodeMixShader')
        bsdf.inputs["Base Color"].default_value = color
        bsdf.inputs["Metallic"].default_value = metallic
        bsdf.inputs["Roughness"].default_value = roughness
        mix_shader.inputs[0].default_value = 1 - alpha  # 透明度控制
        links.new(bsdf.outputs[0], mix_shader.inputs[2])      
        links.new(transparent.outputs[0], mix_shader.inputs[1]) 
        links.new(mix_shader.outputs[0], output.inputs[0])    
        matg.blend_method = 'BLEND'
        matg.shadow_method = 'HASHED'
    return matg