WebRtc06: 音视频数据采集

学而知不足~2025-01-29 17:44

音视频采集API

通过getUserMedia这个API去获取视频音频，

通过constraints这个对象去配置偏好，比如视频宽高、音频降噪等

测试代码

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
    </head>
    <body>
            <video autoplay playsinline id="player"></video>
            <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

var videoplay = document.querySelector('video#player');
function getMediaStream(stream) {
    videoplay.srcObject = stream;
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
    console.log('getUserMedia is not supported');
} else {
    var constraints = {
        video : true,
        audio : true
    }
    navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).catch(handleError);
}

会打开前置摄像头并且输出视频音频

浏览器适配

getUserMedia适配

  • getUserMedia (www)
  • webkitGetUserMedia(google)
  • mozGetUserMedia(firefox)
    每个厂商的API名称都不一样
    比如在前面的index.html中加上下述代码
html 复制代码 
<script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>

获取音视频设备的访问权限

在前面代码的基础上，返回navigator.mediaDevices.enumerateDevices()，从而得到每一个媒体信息

然后通过添加到HTML的选择器中实现打印不同设备信息

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <video autoplay playsinline id="player"></video>
                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
    console.log('getUserMedia is not supported');
} else {
    var constraints = {
        video : true,
        audio : true
    }
    navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
}

音视频采集约束

视频约束

  • width
  • height
  • aspectRatio 宽高比
  • frameRate 帧率
  • facingMode
    • user 前置摄像头
    • environment 后置摄像头
    • left 前置左侧摄像头
    • right 前置右侧摄像头
  • resizeMode

设置constraints中视频的参数如下

js 复制代码 
var constraints = {
        video : {
            width: 1920,
            height: 1080,
            frameRate: 30,
            facingMode: 'environment'
        },
        audio : true
    }

音频约束

  • volume 音量 [0-1.0]
  • sampleRate 采样率
  • sampleSize 采样大小 位深 一般16位
  • echoCancellation 回音消除 true/false
  • autoGainControl 自动增益 true/false 在原有声音的基础上，增加音量
  • noiseSuppression 降噪 true/false
  • latency 延迟大小
  • channelCount 声道数量
  • deviceID
  • groupID 组ID
js 复制代码 
var constraints = {
        video : {
            width: 1920,
            height: 1080,
            frameRate: 30,
            facingMode: 'environment'
        },
        audio : {
            noiseSuppression: true,
            echoCancellation: true
        }
    }

整体例子：切换不同摄像头

js 复制代码 
'use strict'

var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;
        console.log('lai: deviceInfo: ', deviceInfos.label);

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 1920,
                height: 1080,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            audio : {
                noiseSuppression: true,
                echoCancellation: true
            }
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

实战

视频渲染特效

浏览器视频特效

CSS filter: -webkit-filter/filter

不同浏览器的名称不一样，Chrome使用的是-webkit-filter，IE用的是filter

需要知道如何将video和filter关联

特效的底层调用都是OpenGL/Metal等等

常用特效

代码测试

在index.html中添加css和filter的selector，然后在js中指定播放器用的类

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                <video autoplay playsinline id="player"></video>
                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

var filterSelect = document.querySelector('select#filter');

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;
        console.log('lai: deviceInfo: ', deviceInfos.label);

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 1920,
                height: 1080,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            audio : {
                noiseSuppression: true,
                echoCancellation: true
            }
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

从视频中获取图片

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                <video autoplay playsinline id="player"></video>
                <div>
                        <button id="snapshot">Take snapshot</button>
                </div>
                <div>
                        <canvas id="picture"></canvas>
                </div>

                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

// filter
var filterSelect = document.querySelector('select#filter');

// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 1920,
                height: 1080,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            audio : {
                noiseSuppression: true,
                echoCancellation: true
            }
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

// 点击按钮进行截图，但是这里的截图保存到本地是没有滤镜效果的
snapshot.onclick = function() {
    picture.className = filterSelect.value;
    picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}

只采集音频数据

临时屏蔽vidoplayer，设置constraints中的video为false，获取到的媒体流赋值给audioplayer

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                        <!-- controls用于显示控制按钮 -->
                        <audio autoplay controls id="audioplayer" ></audio>
                        <!-- <video autoplay playsinline id="player"></video> -->
                <div>
                        <button id="snapshot">Take snapshot</button>
                </div>
                <div>
                        <canvas id="picture"></canvas>
                </div>

                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

// filter
var filterSelect = document.querySelector('select#filter');

// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;

// var videoplay = document.querySelector('video#player');
// 获取audioplayer
var audioplay = document.querySelector('audio#audioplayer');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    // videoplay.srcObject = stream;
    audioplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            // video : {
            //     width: 1920,
            //     height: 1080,
            //     frameRate: 30,
            //     deviceId : deviceId ? deviceId : undefined
            // },
            video : false,
            audio : true
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

snapshot.onclick = function() {
    picture.className = filterSelect.value;
    picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}

MediaStreamAPI及获取视频约束

MediaStream API

js 复制代码 
MediaStream.addTrack()
MediaStream.removeTrack()
MediaStream.getVideoTracks()
MediaStream.getAudioTracks()

MediaStream事件

js 复制代码 
MediaStream.onaddtrack()
MediaStream.onremovetrack()
MediaStream.onended()

获取视频约束

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                        <!-- controls用于显示控制按钮 -->
                        <!-- <audio autoplay controls id="audioplayer" ></audio> -->
                <table>
                        <tr>
                                <td><video autoplay playsinline id="player"></video></td>
                                <!-- 显示视频约束 -->
                                <td><div id = 'constraints' class='output'></div></td>
                        </tr>
                </table>
                        
                <div>
                        <button id="snapshot">Take snapshot</button>
                </div>
                <div>
                        <canvas id="picture"></canvas>
                </div>

                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

// filter
var filterSelect = document.querySelector('select#filter');

// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;

var videoplay = document.querySelector('video#player');
// 获取audioplayer
// var audioplay = document.querySelector('audio#audioplayer');

var divConstraints = document.querySelector('div#constraints');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;

    var videoTrack = stream.getVideoTracks()[0];
    var videoConstraints = videoTrack.getSettings();
    divConstraints.textContent = JSON.stringify(videoConstraints, null, 2);

    // audioplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 640,
                height: 480,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            // video : false,
            audio : true
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

snapshot.onclick = function() {
    picture.className = filterSelect.value;
    picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}
相关推荐
拾荒的小海螺4 小时前
开源项目：LTX2 高效可控的开源视频生成模型
开源·音视频
EasyGBS12 小时前
视频画面模糊、卡顿、丢失？EasyGBS新增“视频质量诊断”功能，告别人工盯屏
视觉检测·音视频·gb28181·花屏·视频质量诊断·蓝屏检测
zhuweisky13 小时前
ArkTS实现鸿蒙手机视频聊天、屏幕分享（HarmonyOS）
音视频·harmonyos·鸿蒙开发
XHW___00113 小时前
webrtc 关键模块创建的时机
网络·音视频·webrtc
Leinwin14 小时前
VibeVoice-ASR：突破60分钟长音频处理瓶颈，语音识别进入端到端时代
人工智能·音视频·语音识别
EasyDSS15 小时前
直播点播/视频会议EasyDSS一站式视频云平台，全场景视频服务开箱即用
音视频·hls·m3u8·点播技术·流媒体直播
我真会写代码16 小时前
WebSocket：告别轮询，实现Web实时通信 WebRTC：无需插件，实现浏览器端实时音视频通信
网络·websocket·网络协议·webrtc·实时音视频
Guheyunyi16 小时前
什么是安全监测预警系统？应用场景有哪些？
大数据·运维·人工智能·安全·音视频
LittroInno17 小时前
TVMS视频管理平台 —— 目标识别跟踪
人工智能·计算机视觉·音视频
newbiai17 小时前
电商直播AI视频生成工具哪个方便快捷？
人工智能·python·音视频
热门推荐
01GitHub 镜像站点02Claude Code + GLM4.7 避坑指南：解决 Unable to connect to Anthropic services03Vue-skills的中文文档04OpenClaw Chrome扩展使用教程 - 浏览器中继控制05让 Trae IDE 智能体 “读懂”文档 Excel+PDF+DOCX ：mcp-documents-reader 工具使用指南06UV安装并设置国内源07Claude Code Skills 实用使用手册08一文了解国产算子编程语言 TileLang，TileLang 对国产开源生态的影响与启示09Linux下V2Ray安装配置指南10AI 规范驱动开发“三剑客”深度对比：Spec-Kit、Kiro 与 OpenSpec 实战指南