技术背景
我们在对接Windows平台RTSP|RTMP直播播放模块的时候，有开发者提出来这样的技术需求，他们做驾考、全景摄像头、多路会议录制等场景的时候，希望把多路视频流数据，合并到一路保存或者对外推送到RTMP服务。
技术实现
多路RTSP|RTMP流合流，实际上我们2016年就有这块demo，当时合流的数据是本地采集的摄像头或屏幕数据，和外部RTSP、RTMP流，合成后输出（类似于传统意义的连麦操作）。这里大概说下思路，外部的RTSP|RTMP流数据，解码后，把YUV或RGB数据回调上来，然后，按照图层的形式，分别贴摄像头、屏幕数据或解码后的流数据。
本次以四路RTSP摄像头数据合流为例：
开始播放：
ini
/*
* SmartPlayerDemo.cs
* Author: daniusdk.com
* QQ: 89030985
*/
private void btn_play1_Click(object sender, EventArgs e)
{
if (btn_play1.Text == "播放")
{
String url = textBox_url1.Text;
if (!InitCommonSDKParam(player1_handle_, url))
{
MessageBox.Show("设置参数错误!");
return;
}
bool is_support_d3d_render = false;
Int32 in_support_d3d_render = 0;
if (NT.NTBaseCodeDefine.NT_ERC_OK == NTSmartPlayerSDK.NT_SP_IsSupportD3DRender(player1_handle_, playWnd1.Handle, ref in_support_d3d_render))
{
if (1 == in_support_d3d_render)
{
is_support_d3d_render = true;
}
}
if (is_support_d3d_render)
{
// 支持d3d绘制的话，就用D3D绘制
NTSmartPlayerSDK.NT_SP_SetRenderWindow(player1_handle_, playWnd1.Handle);
if (btn_check_render_scale_mode.Checked)
NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player1_handle_, 1);
else
NTSmartPlayerSDK.NT_SP_SetRenderScaleMode(player1_handle_, 0);
}
video_frame_call_back1_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player1_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back1_);
UInt32 ret_start = NTSmartPlayerSDK.NT_SP_StartPlay(player1_handle_);
if (ret_start != 0)
{
MessageBox.Show("播放失败..");
return;
}
btn_play1.Text = "停止";
}
else
{
StopPlayback1();
}
}其中InitCommonSDKParam()主要完成一些初始化参数设置：
scss
private bool InitCommonSDKParam(IntPtr handle, String url) {
if (IntPtr.Zero == handle)
return false;
if (String.IsNullOrEmpty(url))
return false;
Int32 buffer_time = int.Parse(textBox_buffer_time.Text);
NTSmartPlayerSDK.NT_SP_SetBuffer(handle, buffer_time);
// 设置rtsp tcp模式，rtmp不使用, 可以不设置
if (checkBox_rtsp_tcp.Checked)
NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(handle, 1);
else
NTSmartPlayerSDK.NT_SP_SetRTSPTcpMode(handle, 0);
//RTSP timeout设置
Int32 rtsp_timeout = 10;
NTSmartPlayerSDK.NT_SP_SetRtspTimeout(handle, rtsp_timeout);
//RTSP TCP/UDP自动切换设置
Int32 is_auto_switch_tcp_udp = 1;
NTSmartPlayerSDK.NT_SP_SetRtspAutoSwitchTcpUdp(handle, is_auto_switch_tcp_udp);
if (checkBox_mute.Checked)
NTSmartPlayerSDK.NT_SP_SetMute(handle, 1);
else
NTSmartPlayerSDK.NT_SP_SetMute(handle, 0);
if (checkBox_fast_startup.Checked)
NTSmartPlayerSDK.NT_SP_SetFastStartup(handle, 1);
else
NTSmartPlayerSDK.NT_SP_SetFastStartup(handle, 0);
if (checkBox_hardware_decoder.Checked)
{
NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(handle, is_support_h264_hardware_decoder_ ? 1 : 0, 0);
NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(handle, is_support_h265_hardware_decoder_ ? 1 : 0, 0);
}
else
{
NTSmartPlayerSDK.NT_SP_SetH264HardwareDecoder(handle, 0, 0);
NTSmartPlayerSDK.NT_SP_SetH265HardwareDecoder(handle, 0, 0);
}
// 设置是否只解码关键帧
if (btn_check_only_decode_video_key_frame.Checked)
NTSmartPlayerSDK.NT_SP_SetOnlyDecodeVideoKeyFrame(handle, 1);
else
NTSmartPlayerSDK.NT_SP_SetOnlyDecodeVideoKeyFrame(handle, 0);
// 设置低延迟模式
if (checkBox_low_latency.Checked)
NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(handle, 1);
else
NTSmartPlayerSDK.NT_SP_SetLowLatencyMode(handle, 0);
NTSmartPlayerSDK.NT_SP_SetRotation(handle, rotate_degrees_);
NTSmartPlayerSDK.NT_SP_SetAudioVolume(handle, slider_audio_volume.Value);
NTSmartPlayerSDK.NT_SP_SetReportDownloadSpeed(handle, 1, 1);
NTSmartPlayerSDK.NT_SP_SetURL(handle, url);
return true;
}开始播放之前，我们设置YUV数据回调：
ini
video_frame_call_back1_ = new SP_SDKVideoFrameCallBack(SetVideoFrameCallBack);
NTSmartPlayerSDK.NT_SP_SetVideoFrameCallBack(player1_handle_, (Int32)NT.NTSmartPlayerDefine.NT_SP_E_VIDEO_FRAME_FORMAT.NT_SP_E_VIDEO_FRAME_FROMAT_I420, IntPtr.Zero, video_frame_call_back1_);回调处理如下，如果是多个图层，通过推送端，把yuv或rgb数据，投递给推送端，video frame数据回调，可以根据handle区分不同的图层或实例：
ini
public void SetVideoFrameCallBack(IntPtr handle, IntPtr userData, UInt32 status, IntPtr frame)
{
if (frame == IntPtr.Zero)
return;
NT_SP_VideoFrame video_frame = (NT_SP_VideoFrame)Marshal.PtrToStructure(frame, typeof(NT_SP_VideoFrame));
if (publisher_wrapper_ != null) {
int video_layer_index;
if (handle == player_handle_)
video_layer_index = publisher_wrapper_.get_external_video_layer0_index();
else if (handle == player1_handle_)
video_layer_index = publisher_wrapper_.get_external_video_layer1_index();
else if (handle == player2_handle_)
video_layer_index = publisher_wrapper_.get_external_video_layer2_index();
else if (handle == player3_handle_)
video_layer_index = publisher_wrapper_.get_external_video_layer3_index();
else
video_layer_index = -1;
if (video_layer_index > -1)
{
publisher_wrapper_.post_i420_layer_image(video_layer_index, video_frame.plane0_, video_frame.stride0_, video_frame.plane1_, video_frame.stride1_,
video_frame.plane2_, video_frame.stride2_,
video_frame.width_, video_frame.height_);
}
}
}推送端，目前以四路合成为例，另外加个实时文字水印，图层设计如下：
ini
public bool config_layers(bool is_add_rgbx_zero_layer)
{
if (video_option_ != (uint)NTSmartPublisherDefine.NT_PB_E_VIDEO_OPTION.NT_PB_E_VIDEO_OPTION_LAYER)
return false;
if (is_empty_handle())
return false;
int w = video_width_;
int h = video_height_;
if ((w & 0x1) != 0)
--w;
if ((h & 0x1) != 0)
--h;
if (w < 2 || h < 2)
return false;
NTSmartPublisherSDK.NT_PB_ClearLayersConfig(handle_, 0, 0, IntPtr.Zero);
int type, index = 0;
if (is_add_rgbx_zero_layer)
{
NT_PB_RGBARectangleLayerConfig rgba_layer = new NT_PB_RGBARectangleLayerConfig();
type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_RGBA_RECTANGLE;
fill_layer_base(rgba_layer, out rgba_layer.base_, type, index, true, 0, 0, w, h);
rgba_layer.red_ = 0;
rgba_layer.green_ = 0;
rgba_layer.blue_ = 0;
rgba_layer.alpha_ = 255;
if (add_layer_config(rgba_layer, type))
index++;
}
NT_PB_ExternalVideoFrameLayerConfig external_video_layer0 = new NT_PB_ExternalVideoFrameLayerConfig();
type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
fill_layer_base(external_video_layer0, out external_video_layer0.base_, type, index, true, 0, 0, w/2, h/2);
if (add_layer_config(external_video_layer0, type))
external_video_layer0_index_ = index++;
NT_PB_ExternalVideoFrameLayerConfig external_video_layer1 = new NT_PB_ExternalVideoFrameLayerConfig();
type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
fill_layer_base(external_video_layer1, out external_video_layer1.base_, type, index, true, w / 2, 0, w / 2, h / 2);
if (add_layer_config(external_video_layer1, type))
external_video_layer1_index_ = index++;
NT_PB_ExternalVideoFrameLayerConfig external_video_layer2 = new NT_PB_ExternalVideoFrameLayerConfig();
type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
fill_layer_base(external_video_layer2, out external_video_layer2.base_, type, index, true, 0, h / 2, w / 2, h / 2);
if (add_layer_config(external_video_layer2, type))
external_video_layer2_index_ = index++;
NT_PB_ExternalVideoFrameLayerConfig external_video_layer3 = new NT_PB_ExternalVideoFrameLayerConfig();
type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
fill_layer_base(external_video_layer3, out external_video_layer3.base_, type, index, true, w / 2, h / 2, w / 2, h / 2);
if (add_layer_config(external_video_layer3, type))
external_video_layer3_index_ = index++;
//叠加的文本层
NT_PB_ExternalVideoFrameLayerConfig text_layer = new NT_PB_ExternalVideoFrameLayerConfig();
type = (Int32)NTSmartPublisherDefine.NT_PB_E_LAYER_TYPE.NT_PB_E_LAYER_TYPE_EXTERNAL_VIDEO_FRAME;
fill_layer_base(text_layer, out text_layer.base_, type, index, false, w / 2, h / 2, 64, 64);
if (add_layer_config(text_layer, type))
text_layer_index_ = index++;
return index > 0;
}合成后数据，可以对外推送到RTMP服务，也可以注入到本地RTSP服务，或者本地直接录制MP4文件,录制出来四宫格效果如下：
总结
多路RTSP|RTMP数据合流，在多媒体处理、实时监控、驾考、教育等各个行业，应用非常广泛，除了视频外，音频如果需要合成，可以以采集系统扬声器的形式合流出来。多路合流，可以事先做好排版编辑，如果期间不希望显示某一路数据，可以点隐藏图层，实时对图层进行隐藏。感兴趣的开发者，可以单独跟我沟通交流。