rk3588因为其支持8k级视频解码,4k级编码,其强悍的编解码性能常常用来做大屏监控的嵌入式盒子的解决方案,可以部分替代笨重且昂贵的传统视频电视墙的多媒体矩阵服务器,适合小型的监控场景多路监控的应用。
ffmpeg媒体框架的强大的拉流推流以及协议解析组包能力,可以实现多种流的拉取和解析,采用他可以提高程序的兼容性和普适性,通过多线程的方式可以开启多路独立的流的拉取,并实时获取raw数据供解码器解码,为了实现多路并行机制,采用了线程安全的队列实现简单的生产消费模式,可以无阻塞的让数据流在传输和解码,拼接间流畅的运行。
rk的libmpp是瑞芯微的一个开源的通用编解码封装的软件包,实现了强大的零拷贝解码以及rga视频图像的缩放拼接等功能。同时还支持v4l2的采集功能,是一个rk硬件平台理想的音频处理库。利用上述的代码组合可以实现强大的音视频软件功能,兼顾软件生态和灵活控制的多重需求。是硬件播放器,拉流转推,mcu融合的好帮手。



mpp通过dmafd句柄的操作,可以避免在内存和vpu gpu间的拷贝动作,将编解码速度降到5ms以内,满足实时的多路解码融合的要求,通过灵活的编程,分配内部的硬件存储空间,实现yuv大数据量的处理能力。
下面是一个拉流处理的流程框架程序供参考
cpp
// 文件:main.cpp
#include "ffmpeg_input.h"
#include "mpp_decoder.h"
#include "stitch_encoder.h"
#include "ffmpeg_output.h"
#include "config.h"
#include "web_server.h"
#include "status.h"
#include <vector>
#include <iostream>
#include <csignal>
#include <atomic>
#include <spdlog/spdlog.h>
#include <spdlog/sinks/rotating_file_sink.h>
#include <mutex>
std::atomic<bool> g_running{true};
std::atomic<bool> g_reload_config{false};
std::mutex g_log_mutex;
void signal_handler(int) { g_running = false; }
// 状态监控输出
void print_status(const MixerConfig& config) {
std::lock_guard<std::mutex> lock(g_log_mutex);
spdlog::info("当前输入流数: {}", config.inputs.size());
for (size_t i = 0; i < config.inputs.size(); ++i) {
spdlog::info("输入流{}: {} ({}x{}, audio:{})", i, config.inputs[i].url, config.inputs[i].video_width, config.inputs[i].video_height, config.inputs[i].audio);
}
spdlog::info("输出流: {} ({}x{}, {}kbps, {}kbps, fps:{})", config.output.url, config.output.video_width, config.output.video_height, config.output.video_bitrate/1000, config.output.audio_bitrate/1000, config.output.fps);
spdlog::info("布局: {} grid_size:{}", (int)config.layout, config.grid_size);
}
// 音频参数一致性检查
bool check_audio_params(const MixerConfig& config, int& sample_rate, int& channels) {
sample_rate = config.output.audio_bitrate > 0 ? config.output.audio_bitrate : 44100;
channels = 2;
for (const auto& in : config.inputs) {
if (in.audio) {
// 可扩展:从流信息动态获取参数
// 这里只能用配置或默认值
}
}
return true;
}
int main() {
auto logger = spdlog::rotating_logger_mt("mainlog", "logs/streammixer.log", 10*1024*1024, 5);
spdlog::set_default_logger(logger);
MixerConfig config;
if (!load_config("config.json", config)) {
std::cerr << "Failed to load config.json" << std::endl;
return 1;
}
int NUM_STREAMS = config.inputs.size();
int OUTPUT_WIDTH = config.output.video_width;
int OUTPUT_HEIGHT = config.output.video_height;
int audio_sample_rate = config.output.audio_bitrate > 0 ? config.output.audio_bitrate : 44100;
int audio_channels = 2;
check_audio_params(config, audio_sample_rate, audio_channels);
AVSampleFormat audio_fmt = AV_SAMPLE_FMT_FLTP;
// 创建队列
std::vector<SafeQueue<AVPacket*>> video_packet_queues(NUM_STREAMS);
std::vector<SafeQueue<AVPacket*>> audio_packet_queues(NUM_STREAMS);
std::vector<SafeQueue<MppFrame>> decoded_frame_queues(NUM_STREAMS);
std::vector<SafeQueue<AVFrame*>> audio_frame_queues(NUM_STREAMS);
SafeQueue<MppPacket> output_packet_queue;
SafeQueue<AVFrame*> mixed_audio_queue;
SafeQueue<AVPacket*> mixed_audio_packet_queue;
// 输入URL列表
std::vector<std::string> input_urls;
for (auto& in : config.inputs) input_urls.push_back(in.url);
// 初始化各模块
FFmpegInput* input = new FFmpegInput(NUM_STREAMS, video_packet_queues, audio_packet_queues, config.inputs);
input->init(input_urls);
MppDecoder* video_decoder = new MppDecoder(NUM_STREAMS, video_packet_queues, decoded_frame_queues);
video_decoder->init();
// 音频解码(每路一个线程)
std::vector<std::unique_ptr<AudioDecoder>> audio_decoders;
for (int i = 0; i < NUM_STREAMS; ++i) {
audio_decoders.emplace_back(new AudioDecoder("aac", audio_packet_queues[i], audio_frame_queues[i], audio_sample_rate, audio_channels, audio_fmt));
}
// 音频混音
AudioMixer* audio_mixer = new AudioMixer(NUM_STREAMS, audio_frame_queues, mixed_audio_queue, audio_sample_rate, audio_channels, audio_fmt);
// 音频编码
AudioEncoder* audio_encoder = new AudioEncoder("aac", audio_sample_rate, audio_channels, mixed_audio_queue, mixed_audio_packet_queue);
StitchEncoder* encoder = new StitchEncoder(NUM_STREAMS, decoded_frame_queues, output_packet_queue, config.layout, config.grid_size, OUTPUT_WIDTH, OUTPUT_HEIGHT);
encoder->init();
FFmpegOutput* output = new FFmpegOutput(config.output.url, output_packet_queue, mixed_audio_packet_queue, config.output);
output->init();
WebServer web("config.json", 8080);
web.start();
// 启动所有线程
try {
input->start();
video_decoder->start();
for (auto& dec : audio_decoders) dec->start();
audio_mixer->start();
audio_encoder->start();
encoder->start();
output->start();
} catch (const std::exception& e) {
spdlog::error("线程启动异常: {}", e.what());
return 2;
}
std::signal(SIGINT, signal_handler);
print_status(config);
while (g_running) {
std::this_thread::sleep_for(std::chrono::seconds(1));
print_status(config);
if (g_reload_config) {
g_reload_config = false;
spdlog::warn("配置热加载: 停止所有线程并重建...");
// 停止所有线程
output->stop(); encoder->stop(); audio_encoder->stop(); audio_mixer->stop();
for (auto& dec : audio_decoders) dec->stop();
video_decoder->stop(); input->stop();
// 资源释放
delete output; delete encoder; delete audio_encoder; delete audio_mixer; delete video_decoder; delete input;
audio_decoders.clear();
// 重新加载配置
MixerConfig new_config;
if (load_config("config.json", new_config)) {
config = new_config;
NUM_STREAMS = config.inputs.size();
OUTPUT_WIDTH = config.output.video_width;
OUTPUT_HEIGHT = config.output.video_height;
check_audio_params(config, audio_sample_rate, audio_channels);
// 重新创建队列
video_packet_queues.assign(NUM_STREAMS, SafeQueue<AVPacket*>());
audio_packet_queues.assign(NUM_STREAMS, SafeQueue<AVPacket*>());
decoded_frame_queues.assign(NUM_STREAMS, SafeQueue<MppFrame>());
audio_frame_queues.assign(NUM_STREAMS, SafeQueue<AVFrame*>());
// 重新初始化模块
input = new FFmpegInput(NUM_STREAMS, video_packet_queues, audio_packet_queues, config.inputs);
input->init(input_urls);
video_decoder = new MppDecoder(NUM_STREAMS, video_packet_queues, decoded_frame_queues);
video_decoder->init();
for (int i = 0; i < NUM_STREAMS; ++i) {
audio_decoders.emplace_back(new AudioDecoder("aac", audio_packet_queues[i], audio_frame_queues[i], audio_sample_rate, audio_channels, audio_fmt));
}
audio_mixer = new AudioMixer(NUM_STREAMS, audio_frame_queues, mixed_audio_queue, audio_sample_rate, audio_channels, audio_fmt);
audio_encoder = new AudioEncoder("aac", audio_sample_rate, audio_channels, mixed_audio_queue, mixed_audio_packet_queue);
encoder = new StitchEncoder(NUM_STREAMS, decoded_frame_queues, output_packet_queue, config.layout, config.grid_size, OUTPUT_WIDTH, OUTPUT_HEIGHT);
encoder->init();
output = new FFmpegOutput(config.output.url, output_packet_queue, mixed_audio_packet_queue, config.output);
output->init();
print_status(config);
// 启动所有线程
try {
input->start(); video_decoder->start();
for (auto& dec : audio_decoders) dec->start();
audio_mixer->start(); audio_encoder->start(); encoder->start(); output->start();
} catch (const std::exception& e) {
spdlog::error("热加载线程启动异常: {}", e.what());
}
} else {
spdlog::error("配置热加载失败,未能重新初始化");
}
}
}
// 程序退出前
web.stop();
output->stop(); encoder->stop(); audio_encoder->stop(); audio_mixer->stop();
for (auto& dec : audio_decoders) dec->stop();
video_decoder->stop(); input->stop();
delete output; delete encoder; delete audio_encoder; delete audio_mixer; delete video_decoder; delete input;
audio_decoders.clear();
spdlog::info("Stream stopped");
}
视频解码
cpp
void MppDecoder::run(int stream_idx) {
auto& ctx = contexts[stream_idx];
auto& in_queue = input_queues[stream_idx];
auto& out_queue = output_queues[stream_idx];
while (running) {
AVPacket* av_pkt = nullptr;
in_queue.pop(av_pkt);
if (!av_pkt) {
// 输出黑帧
MppFrame black_frame = nullptr;
mpp_frame_init(&black_frame);
mpp_frame_set_width(black_frame, 640); // 建议用实际分辨率
mpp_frame_set_height(black_frame, 360);
mpp_frame_set_fmt(black_frame, MPP_FMT_YUV420SP);
MppBuffer buf = nullptr;
size_t buf_size = 640 * 360 * 3 / 2;
mpp_buffer_get(NULL, &buf, buf_size);
memset(mpp_buffer_get_ptr(buf), 0, buf_size);
mpp_frame_set_buffer(black_frame, buf);
out_queue.push(std::move(black_frame));
continue;
}
// 将FFmpeg包转换为MPP包
MppPacket mpp_pkt = nullptr;
mpp_packet_init(&mpp_pkt, av_pkt->data, av_pkt->size);
mpp_packet_set_pts(mpp_pkt, av_pkt->pts);
MPP_RET ret = ctx.mpi->decode_put_packet(ctx.ctx, mpp_pkt);
if (ret != MPP_OK) {
mpp_packet_deinit(&mpp_pkt);
av_packet_free(&av_pkt);
continue;
}
MppFrame frame = nullptr;
do {
ret = ctx.mpi->decode_get_frame(ctx.ctx, &frame);
if (ret != MPP_OK || !frame) break;
if (mpp_frame_get_info_change(frame)) {
// 处理格式变化
ctx.mpi->control(ctx.ctx, MPP_DEC_SET_INFO_CHANGE_READY, NULL);
} else if (mpp_frame_get_eos(frame)) {
// 结束流
} else {
out_queue.push(std::move(frame));
frame = nullptr;
}
} while (1);
mpp_packet_deinit(&mpp_pkt);
av_packet_free(&av_pkt);
// 解码失败时也输出黑帧
if (ret != MPP_OK) {
MppFrame black_frame = nullptr;
mpp_frame_init(&black_frame);
mpp_frame_set_width(black_frame, 640);
mpp_frame_set_height(black_frame, 360);
mpp_frame_set_fmt(black_frame, MPP_FMT_YUV420SP);
MppBuffer buf = nullptr;
size_t buf_size = 640 * 360 * 3 / 2;
mpp_buffer_get(NULL, &buf, buf_size);
memset(mpp_buffer_get_ptr(buf), 0, buf_size);
mpp_frame_set_buffer(black_frame, buf);
out_queue.push(std::move(black_frame));
continue;
}
}
}