OpenCV：实时视频防抖，python版（改进连续帧处理）

• 改进思路

上个版本的代码，每30帧就全清。显然不符合我们要求。

我们期望：分析当前帧的前30帧的情况。

• 代码

同时去掉了无用的gray数组。

import numpy as np
import cv2
import time



# 给画面画网格线
def video_grid(frame, width, height, direction, color):
    if (not merge_video):
        return

    if (direction):
        for i in range(1, int(width/grid_size)+1):
            cv2.line(frame,
                     (int(grid_size*i), int(0)          ),
                     (int(grid_size*i), int(height     )),
                     color, 2)
    else:
        for i in range(1, int(height/grid_size)+1):
            cv2.line(frame,
                     (int(0          ), int(grid_size*i)),
                     (int(width      ), int(grid_size*i)),
                     color, 2)

def video_circle(frame, width, height, color):
    cv2.circle(frame,
               (int(width/2), int(height/2)),
               5, color, 3)

    cv2.circle(frame,
                   (int(width/2), int(height/2)),
                   grid_size, color, 2)
                   
    size = min(width, height)
    for i in range(1, int(size/grid_size)+1):
        cv2.circle(frame,
                   (int(width/2), int(height/2)),
                   grid_size, color, 2)
                   
# 标出匹配的关键点
def video_keypoints(frame, matches, keypoints, check_x, check_y):
    for match in matches:
        curr_point = keypoints[match.trainIdx].pt
        cv2.circle(frame,
                   (int(curr_point[0]+check_x), int(curr_point[1]+check_y)),
                   3, (0, 255, 0), 2)

# 文件名
def get_filename():
    filename = 'stab'
    if (merge_video):
        if (merge_horizontal):
            filename += '_h'
        else:
            filename += '_v'

    filename += '.mp4'
    return filename

# 计算剪裁位置，避免越界
def get_dst_pos(crop_p, mean_motion_p):
    dst_p = int(crop_p + mean_motion_p)
    if (dst_p < 0):
        dst_p = 0
    elif (dst_p > crop_p*2):
        dst_p = crop_p*2

    return dst_p


def movingAverage(curve, radius):
    window_size = 2 * radius + 1
    # 定义过滤器
    f = np.ones(window_size) / window_size
    # 为边界添加填充
    curve_pad = np.pad(curve, (radius, radius), 'edge')
    # 应用卷积
    curve_smoothed = np.convolve(curve_pad, f, mode='same')
    # 删除填充
    curve_smoothed = curve_smoothed[radius:-radius]
    # 返回平滑曲线
    return curve_smoothed


def smooth(trajectory):
    smoothed_trajectory = np.copy(trajectory)
    # 过滤x, y和角度曲线
    for i in range(3):
        smoothed_trajectory[:, i] = movingAverage(
            trajectory[:, i], radius=SMOOTHING_RADIUS)

    return smoothed_trajectory


def fixBorder(frame):
    s = frame.shape
    T = cv2.getRotationMatrix2D((s[1] / 2, s[0] / 2), 0, ZOOM_RATIO)
    frame = cv2.warpAffine(frame, T, (s[1], s[0]))
    return frame

# 在不移动中心的情况下，将图像缩放
ZOOM_RATIO = 1.05

FRAME_BUFFER_COUNT = 30

# 尺寸越大，视频越稳定，但对突然平移的反应越小
SMOOTHING_RADIUS = 50

# 是否合并视频
merge_video      = True
# 合并视频时，水平或垂直
merge_horizontal = True

# 网格线间隔
grid_size        = 100


# 读取输入视频
cap = cv2.VideoCapture('../test-1920X1080.mp4')

# 获取视频流的宽度和高度
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
# 获取每秒帧数(fps)
fps = cap.get(cv2.CAP_PROP_FPS)

src_width = w
src_height= h
# 只检测中心位置的区域。
check_n = 1/5
check_w = int( src_width *check_n)
check_h = int( src_height*check_n)
check_x = int((src_width -check_w)/2)
check_y = int((src_height-check_h)/2)

# 剪裁范围
crop_n  = 1/5
crop_x  = int(src_width *   crop_n/2)
crop_y  = int(src_height*   crop_n/2)
crop_w  = int(src_width *(1-crop_n) )
crop_h  = int(src_height*(1-crop_n) )

write_w = w
write_h = h
if (merge_video):
    if (merge_horizontal):
        write_w += w
    else:
        write_h += h
        
# 设置视频输出格式
fourcc = cv2.VideoWriter_fourcc(*'mp4v')
out    = cv2.VideoWriter(get_filename(), fourcc, cap.get(cv2.CAP_PROP_FPS), (write_w, write_h))

# # 得到帧数
# n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

# 预定义转换numpy矩阵
# transforms = np.zeros((n_frames - 1, 3), np.float32)

# 准备存储
transforms = np.zeros((FRAME_BUFFER_COUNT, 3), np.float32)

work_cost      = 0
frame_counter  = 0
buffer_pointer = 0
while cap.isOpened():

    # print(k)

    # 读取一帧
    success, curr = cap.read()

    # 耗时开始
    frame_counter += 1
    current_time = time.time()
    
    # 是否还有下一帧，关闭
    if not success:
        break
 
    video_circle(curr, w, h, (0, 0, 255))
    # 转换为灰度图
    curr_gray = cv2.cvtColor(curr, cv2.COLOR_BGR2GRAY)

    if buffer_pointer >= 2:
        # 检测前一帧的特征点
        prev_pts = cv2.goodFeaturesToTrack(prev_gray,
                                           maxCorners=200,
                                           qualityLevel=0.01,
                                           minDistance=30,
                                           blockSize=3)
        # 计算光流(即轨迹特征点) 前一张 当前张 前一张特征
        curr_pts, status, err = cv2.calcOpticalFlowPyrLK(prev_gray, curr_gray, prev_pts, None)

        # 检查完整性
        assert prev_pts.shape == curr_pts.shape

        # 只过滤有效点
        idx = np.where(status == 1)[0]
        prev_pts = prev_pts[idx]
        curr_pts = curr_pts[idx]

        # 找到变换矩阵
        m, inlier = cv2.estimateAffine2D(prev_pts, curr_pts)

        # 提取traslation
        dx = m[0, 2]
        dy = m[1, 2]

        # 提取旋转角
        da = np.arctan2(m[1, 0], m[0, 0])

        # 存储转换
        transforms[buffer_pointer] = [dx, dy, da]

        # cv2.imshow("B", curr_gray)
        # cv2.waitKey(1)

    if buffer_pointer >= 3:
        # 使用累积变换和计算轨迹
        trajectory = np.cumsum(transforms, axis=0)

        # 创建变量来存储平滑的轨迹
        smoothed_trajectory = smooth(trajectory)

        # 计算smoothed_trajectory与trajectory的差值
        difference = smoothed_trajectory - trajectory

        # 计算更新的转换数组
        transforms_smooth = transforms + difference

        # 从新的转换数组中提取转换
        dx = transforms_smooth[buffer_pointer+1, 0]
        dy = transforms_smooth[buffer_pointer+1, 1]
        da = transforms_smooth[buffer_pointer+1, 2]
        # print('frame_counter=', frame_counter, 'dx=', int(dx), 'dy=', int(dy), 'da=', int(da*360))
        
        # 根据新的值重构变换矩阵
        m = np.zeros((2, 3), np.float32)
        m[0, 0] =  np.cos(da)
        m[0, 1] = -np.sin(da)
        m[1, 0] =  np.sin(da)
        m[1, 1] =  np.cos(da)
        m[0, 2] =  dx
        m[1, 2] =  dy

        # 应用仿射包装到给定的框架
        frame_stabilized = cv2.warpAffine(curr, m, (w, h))

        # Fix border artifacts
        frame_stabilized = fixBorder(frame_stabilized)
        video_circle(frame_stabilized, w, h, (0, 255, 255))
        # cv2.imshow("B", frame_stabilized)
        # cv2.waitKey(1)
        
        if (merge_horizontal):
            out_frame = cv2.hconcat([curr, frame_stabilized])
        else:
            out_frame = cv2.vconcat([curr, frame_stabilized])
        
        work_cost += (time.time()-current_time)
        
        # video_grid(out_frame, write_w, write_h, (not merge_horizontal), (0, 0, 255))
        out.write(out_frame)
        
    buffer_pointer += 1
    # transforms_smooth use FRAME_BUFFER_COUNT-1
    if buffer_pointer >= FRAME_BUFFER_COUNT-1:
        buffer_pointer = FRAME_BUFFER_COUNT-2
        for i in range(0, FRAME_BUFFER_COUNT-2):
            transforms[i] = transforms[i+1]

    prev_gray = curr_gray

print('cost per frame(ms)=', (work_cost/frame_counter*1000))
# 释放视频读取和写入对象
cap.release()
out.release()
# 关闭所有OpenCV窗口
cv2.destroyAllWindows()