WebRtc06: 音视频数据采集

学而知不足~2025-01-29 17:44

音视频采集API

通过getUserMedia这个API去获取视频音频，

通过constraints这个对象去配置偏好，比如视频宽高、音频降噪等

测试代码

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
    </head>
    <body>
            <video autoplay playsinline id="player"></video>
            <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

var videoplay = document.querySelector('video#player');
function getMediaStream(stream) {
    videoplay.srcObject = stream;
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
    console.log('getUserMedia is not supported');
} else {
    var constraints = {
        video : true,
        audio : true
    }
    navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).catch(handleError);
}

会打开前置摄像头并且输出视频音频

浏览器适配

getUserMedia适配

  • getUserMedia (www)
  • webkitGetUserMedia(google)
  • mozGetUserMedia(firefox)
    每个厂商的API名称都不一样
    比如在前面的index.html中加上下述代码
html 复制代码 
<script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>

获取音视频设备的访问权限

在前面代码的基础上，返回navigator.mediaDevices.enumerateDevices()，从而得到每一个媒体信息

然后通过添加到HTML的选择器中实现打印不同设备信息

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <video autoplay playsinline id="player"></video>
                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
    console.log('getUserMedia is not supported');
} else {
    var constraints = {
        video : true,
        audio : true
    }
    navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
}

音视频采集约束

视频约束

  • width
  • height
  • aspectRatio 宽高比
  • frameRate 帧率
  • facingMode
    • user 前置摄像头
    • environment 后置摄像头
    • left 前置左侧摄像头
    • right 前置右侧摄像头
  • resizeMode

设置constraints中视频的参数如下

js 复制代码 
var constraints = {
        video : {
            width: 1920,
            height: 1080,
            frameRate: 30,
            facingMode: 'environment'
        },
        audio : true
    }

音频约束

  • volume 音量 [0-1.0]
  • sampleRate 采样率
  • sampleSize 采样大小 位深 一般16位
  • echoCancellation 回音消除 true/false
  • autoGainControl 自动增益 true/false 在原有声音的基础上，增加音量
  • noiseSuppression 降噪 true/false
  • latency 延迟大小
  • channelCount 声道数量
  • deviceID
  • groupID 组ID
js 复制代码 
var constraints = {
        video : {
            width: 1920,
            height: 1080,
            frameRate: 30,
            facingMode: 'environment'
        },
        audio : {
            noiseSuppression: true,
            echoCancellation: true
        }
    }

整体例子：切换不同摄像头

js 复制代码 
'use strict'

var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;
        console.log('lai: deviceInfo: ', deviceInfos.label);

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 1920,
                height: 1080,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            audio : {
                noiseSuppression: true,
                echoCancellation: true
            }
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

实战

视频渲染特效

浏览器视频特效

CSS filter: -webkit-filter/filter

不同浏览器的名称不一样，Chrome使用的是-webkit-filter，IE用的是filter

需要知道如何将video和filter关联

特效的底层调用都是OpenGL/Metal等等

常用特效

代码测试

在index.html中添加css和filter的selector，然后在js中指定播放器用的类

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                <video autoplay playsinline id="player"></video>
                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

var filterSelect = document.querySelector('select#filter');

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;
        console.log('lai: deviceInfo: ', deviceInfos.label);

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 1920,
                height: 1080,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            audio : {
                noiseSuppression: true,
                echoCancellation: true
            }
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

从视频中获取图片

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                <video autoplay playsinline id="player"></video>
                <div>
                        <button id="snapshot">Take snapshot</button>
                </div>
                <div>
                        <canvas id="picture"></canvas>
                </div>

                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

// filter
var filterSelect = document.querySelector('select#filter');

// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;

var videoplay = document.querySelector('video#player');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 1920,
                height: 1080,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            audio : {
                noiseSuppression: true,
                echoCancellation: true
            }
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

// 点击按钮进行截图，但是这里的截图保存到本地是没有滤镜效果的
snapshot.onclick = function() {
    picture.className = filterSelect.value;
    picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}

只采集音频数据

临时屏蔽vidoplayer，设置constraints中的video为false，获取到的媒体流赋值给audioplayer

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                        <!-- controls用于显示控制按钮 -->
                        <audio autoplay controls id="audioplayer" ></audio>
                        <!-- <video autoplay playsinline id="player"></video> -->
                <div>
                        <button id="snapshot">Take snapshot</button>
                </div>
                <div>
                        <canvas id="picture"></canvas>
                </div>

                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

// filter
var filterSelect = document.querySelector('select#filter');

// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;

// var videoplay = document.querySelector('video#player');
// 获取audioplayer
var audioplay = document.querySelector('audio#audioplayer');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    // videoplay.srcObject = stream;
    audioplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            // video : {
            //     width: 1920,
            //     height: 1080,
            //     frameRate: 30,
            //     deviceId : deviceId ? deviceId : undefined
            // },
            video : false,
            audio : true
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

snapshot.onclick = function() {
    picture.className = filterSelect.value;
    picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}

MediaStreamAPI及获取视频约束

MediaStream API

js 复制代码 
MediaStream.addTrack()
MediaStream.removeTrack()
MediaStream.getVideoTracks()
MediaStream.getAudioTracks()

MediaStream事件

js 复制代码 
MediaStream.onaddtrack()
MediaStream.onremovetrack()
MediaStream.onended()

获取视频约束

index.html

html 复制代码 
<html>
    <head>
            <title>WebRtc capture video and audio</title>
            <style>
                .none {
                        -webkit-filter: none;
                }
                .blur {
                        -webkit-filter: blur(3px);
                }
                .grayscale {
                        -webkit-filter:grayscale(1);
                }
                .invert {
                        -webkit-filter: invert(1);
                }
                .sepia {
                        -webkit-filter:sepia(1);
                }
            </style>
    </head>
    <body>
                <div>
                        <label>audioSource:</label>
                        <select id="audioSource"></select>
                </div>
                <div>
                        <label>audioOutput:</label>
                        <select id="audioOutput"></select>
                </div>
                <div>
                        <label>videoSource:</label>
                        <select id="videoSource"></select>
                </div>
                <div>
                        <label>filter:</label>
                        <select id="filter">
                                <option value="none"> None</option>
                                <option value="blur"> blur</option>
                                <option value="grayscale"> grayscale</option>
                                <option value="invert"> invert</option>
                                <option value="sepia"> sepia</option>

                        </select>
                </div>
                        <!-- controls用于显示控制按钮 -->
                        <!-- <audio autoplay controls id="audioplayer" ></audio> -->
                <table>
                        <tr>
                                <td><video autoplay playsinline id="player"></video></td>
                                <!-- 显示视频约束 -->
                                <td><div id = 'constraints' class='output'></div></td>
                        </tr>
                </table>
                        
                <div>
                        <button id="snapshot">Take snapshot</button>
                </div>
                <div>
                        <canvas id="picture"></canvas>
                </div>

                <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script>
                <script src="./js/client.js"></script>
    </body>
</html>

client.js

js 复制代码 
'use strict'

// devices
var audioSource = document.querySelector('select#audioSource');
var audioOutput = document.querySelector('select#audioOutput');
var videoSource = document.querySelector('select#videoSource');

// filter
var filterSelect = document.querySelector('select#filter');

// picture
var snapshot = document.querySelector('button#snapshot');
var picture = document.querySelector('canvas#picture');
picture.width = 320;
picture.height = 240;

var videoplay = document.querySelector('video#player');
// 获取audioplayer
// var audioplay = document.querySelector('audio#audioplayer');

var divConstraints = document.querySelector('div#constraints');

function getDevice(deviceInfos) {
    deviceInfos.forEach(function(deviceInfos) {
        var option = document.createElement('option');
        option.text = deviceInfos.label;
        option.value = deviceInfos.deviceId;

        if (deviceInfos.kind == 'audioinput') {
            audioSource.appendChild(option);
        } else if (deviceInfos.kind == 'audiooutput') {
            audioOutput.appendChild(option);
        } else if (deviceInfos.kind == 'videoinput') {
            videoSource.appendChild(option);
        }
    });
}

function getMediaStream(stream) {
    videoplay.srcObject = stream;

    var videoTrack = stream.getVideoTracks()[0];
    var videoConstraints = videoTrack.getSettings();
    divConstraints.textContent = JSON.stringify(videoConstraints, null, 2);

    // audioplay.srcObject = stream;
    return navigator.mediaDevices.enumerateDevices();
}
function handleError(err) {
    console.log('getUserMedia error: ', err);
}

function start() {
    if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
        console.log('getUserMedia is not supported');
    } else {
        var deviceId = videoSource.value;
        var constraints = {
            video : {
                width: 640,
                height: 480,
                frameRate: 30,
                deviceId : deviceId ? deviceId : undefined
            },
            // video : false,
            audio : true
        }
        navigator.mediaDevices.getUserMedia(constraints).then(getMediaStream).then(getDevice).catch(handleError);
    }
}
start();

videoSource.onchange = start;

filterSelect.onchange = function() {
    videoplay.className = filterSelect.value;
}

snapshot.onclick = function() {
    picture.className = filterSelect.value;
    picture.getContext('2d').drawImage(videoplay, 0, 0, picture.width, picture.height);
}
相关推荐
小gpt&3 小时前
01 音视频知识学习(视频)
c++·qt·学习·ffmpeg·音视频
TSINGSEE3 小时前
AI+视频监控电力巡检：EasyCVR视频中台方案如何赋能电力行业智能化转型
服务器·人工智能·音视频
学习嵌入式的小羊~3 小时前
读取解码视频数据并写入共享内存
音视频
Black蜡笔小新3 小时前
WebRTC技术在音视频处理上的难点剖析：EasyRTC嵌入式视频通话SDK的优化策略
人工智能·大模型·音视频·webrtc
TSINGSEE3 小时前
嵌入式音视频通话SDK组件EasyRTC：全平台设备兼容，智能硬件里的WebRTC调用实践
网络协议·音视频·webrtc·实时音视频·p2p·智能硬件
EasyCVR3 小时前
EasyRTC嵌入式音视频通话SDK：基于纯C语言的跨平台实时通信系统设计与实践
linux·c语言·开发语言·音视频·webrtc·h.265
EasyCVR3 小时前
JavaScript API与WebRTC技术解析：EasyRTC嵌入式视频通话SDK的实现
javascript·音视频·webrtc
Yeauty4 小时前
三分钟掌握音视频处理 | 在 Rust 中优雅地集成 FFmpeg
rust·ffmpeg·音视频
coder阿龙7 小时前
【UNIAPP】获取视频的第一帧作为封面（基于视频URL，Canvas）复制即用
前端·uni-app·音视频
羑悻的小杀马特11 小时前
蓝耘携手通义万象 2.1 图生视频：开启创意无限的共享新时代
服务器·nginx·音视频·ai大模型·蓝耘·通义万象 2.1
热门推荐
01如何在WPS和Word/Excel中直接使用DeepSeek功能02DeepSeek各版本说明与优缺点分析03本地部署DeepSeek教程（Mac版本）04从零安装 LLaMA-Factory 微调 Qwen 大模型成功及所有的坑05如何本地部署AI智能体平台，带你手搓一个AI Agent06DeepSeek R1本地化部署 Ollama + Chatbox 打造最强 AI 工具07DeepSeek RAGFlow构建本地知识库系统08Windows 11 安装 Dify 完整指南 非docker环境09本地化部署AI知识库：基于Ollama+DeepSeek+AnythingLLM保姆级教程10网络安全 - DDoS 攻击原理 + 实验