基于H5的摄像头视频数据流采集
最近，为了⽀持部门团队的项⽬，通过H5实现摄像头的视频流数据的捕获，抓取到视频流后，传输到视频识别服务器进⾏后续的逻辑处理。
视频数据的采集过程，其实是⽐较没有谱的过程，因为之前没有研究过HTML5操控摄像头并取视频流。
这个任务的实现逻辑，前端搭建⼀个Java的⼩Web应⽤，H5视频采集之后，通过WebSocket的⽅式，将视频流数据传递到Java的web⼩应⽤后台，然后从后台向视频识别服务
器通过UDP传递视频数据。基本的架构如下图：
接下来，上前端页⾯以及代码. ⼤体说下，我的软件架构，jersey2 + freemarker + spring。
前端页⾯：
<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes">
<meta name="mobile-web-app-capable" content="yes">
<meta id="theme-color" name="theme-color" content="#fff">
<base target="_blank">
<title>Media Recorder API Demo</title>
<link rel="stylesheet" href="${basePath}/css/video/main.css" />        #basePath是Web项⽬的根地址，例如 10.90.9.20:9080/RDConsumer
<style>
a#downloadLink {
display: block;
margin: 00 1em 0;
min-height: 1.2em;
}
p#data {
min-height: 6em;
}
</style>
</head>
<body>
<div id="container">
<div style = "text-align:center;">
<h1>Media Recorder API Demo </h1>
<h2>Record a 640x480 video using the media recorder API implemented in Firefox and Chrome</h2>
<video controls autoplay></video><br>
<button id="rec" onclick="onBtnRecordClicked()">Record</button>
<button id="pauseRes"  onclick="onPauseResumeClicked()" disabled>Pause</button>
<button id="stop"  onclick="onBtnStopClicked()" disabled>Stop</button>
</div>
<a id="downloadLink" download="mediarecorder.webm" name="mediarecorder.webm" href></a>
<p id="data"></p>
<script src="${basePath}/js/jquery-1.11.1.min.js"></script>
<script src="${basePath}/js/video/main.js"></script>
<h2>Works on:</h2>
<p><ul><li>Firefox 30 and up</li><li>Chrome 47,48 (video only, enable <em>experimental Web Platform features</em> at  <a href="chrome://flags/#enable-experimental-web-platform-features">chrome://flags</a>)</li><li>Chrome 49+</li></ul> <h2>
<span >Issues:</span>
<p><ul><li>Pause does not stop audio recording on Chrome 49,50</li></ul></p>
<h2>Containers & codecs:</h2>
<p><table >
<thead>
<tr>
<th> </th><th>Chrome 47</th><th>Chrome 48</th><th>Chrome 49+</th><th>Chrome 52+</th><th>Firefox 30+</th>
</tr>
</thead>
<tbody>
<tr>
<td><strong>Container</strong></td><td>webm</td><td>webm</td><td>webm</td><td>webm</td><td>webm</td>
</tr>
<tr>
<td><strong>Video</strong></td><td>VP8</td><td>VP8</td><td>VP8/VP9</td><td>VP8/VP9/H264</td><td>VP8</td>
</tr>
<tr>
<td><strong>Audio</strong></td><td>none</td><td>none</td><td>Opus @ 48kHz</td><td>Opus @ 48kHz</td><td>Vorbis @ 44.1 kHz</td>
</tr>
</tbody>
</table>
</p>
<h2>Links:</h2>
<p>
<ul>
<li>Article: <a target="_blank" href="addpipe/blog/mediarecorder-api/">addpipe/blog/mediarecorder-api/</a></li>
<li>GitHub: <a target="_blank" href="github/addpipe/Media-Recorder-API-Demo">gi
thub/addpipe/Media-Recorder-API-Demo</a></li>
<li>W3C Draft: <a target="_blank"  href="w3c.github.io/mediacapture-record/MediaRecorder.html">w3c.github.io/mediacapture-record/MediaRecorder.html</a></li>
<li>Media Recorder API at 65% penetration thanks to Chrome: <a target="_blank" href="addpipe/blog/media-recorder-api-is-now-supported-by-65-of-all-desktop-internet-users/">addpipe/blog/media-recorder-api-is-now-su    </ul>
</p>
</div>
</body>
</html>
前端界⾯的效果图：
JS的代码（重点之⼀在这个JS⾥⾯的红⾊部分，下⾯代码是main.js的正⽂内容）：
'use strict';
/* globals MediaRecorder */
// Spec is at /hg/dap/raw-file/tip/media-stream-capture/RecordingProposal.html
if(getBrowser() == "Chrome"){
var constraints = {"audio": true, "video": {  "mandatory": {  "minWidth": 640,  "maxWidth": 640, "minHeight": 480,"maxHeight": 480 }, "optional": [] } };//Chrome
}else if(getBrowser() == "Firefox"){
var constraints = {audio: false, video: {  width: { min: 640, ideal: 640, max: 640 },  height: { min: 480, ideal: 480, max: 480 }}}; //Firefox
}
var recBtn = document.querySelector('button#rec');
var pauseResBtn = document.querySelector('button#pauseRes');
var stopBtn = document.querySelector('button#stop');
var videoElement = document.querySelector('video');
var dataElement = document.querySelector('#data');
var downloadLink = document.querySelector('a#downloadLink');
function errorCallback(error){
console.log('UserMedia error: ', error);
}
/*
var mediaSource = new MediaSource();
mediaSource.addEventListener('sourceopen', handleSourceOpen, false);
var sourceBuffer;
*/
var mediaRecorder;
var chunks = [];
var count = 0;
var wsurl = "ws://10.90.9.20:9080/RDConsumer/websocket"
var ws = null;
function createWs(){
var url = wsurl;
if ('WebSocket'in window) {
ws = new WebSocket(url);
} else if ('MozWebSocket'in window) {
ws = new MozWebSocket(url);
} else {
console.log("您的浏览器不⽀持WebSocket。");
return ;
}
}
function init() {
if (ws != null) {
console.log("现已连接");
return ;
}
createWs();
//设置发信息送类型为：ArrayBuffer
ws.binaryType = "arraybuffer";
}
console.log(String());
}
console.log("onclose: closed");
ws = null;
createWs(); //这个函数在这⾥之所以再次调⽤，是为了解决视频传输的过程中突发的连接断开问题。
}
console.log("onerror: error");
ws = null;
createWs(); //同上⾯的解释
}
}
$(document).ready(function(){
init();
})
function startRecording(stream) {
log('');
if (typeof MediaRecorder.isTypeSupported == 'function'){
/*
MediaRecorder.isTypeSupported is a function announced in le/web/updates/2016/01/mediarecorder and later introduced in the MediaRecorder API spec /TR/mediastream-recording/ */
//这⾥涉及到视频的容器以及编解码参数，这个与浏览器有密切的关系
if (MediaRecorder.isTypeSupported('video/webm;codecs=vp9')) {
var options = {mimeType: 'video/webm;codecs=h264'};
} else if (MediaRecorder.isTypeSupported('video/webm;codecs=h264')) {
var options = {mimeType: 'video/webm;codecs=h264'};
} else  if (MediaRecorder.isTypeSupported('video/webm;codecs=vp8')) {
var options = {mimeType: 'video/webm;codecs=vp8'};
}
log('Using '+options.mimeType);
mediaRecorder = new MediaRecorder(stream, options);
}else{
log('isTypeSupported is not supported, using default codecs for browser');
mediaRecorder = new MediaRecorder(stream);
}
mediaRecorder.start(10);
var url = window.URL || window.webkitURL;
videoElement.src = url ? ateObjectURL(stream) : stream;
videoElement.play();
//这个地⽅，是视频数据捕获好了后，会触发MediaRecorder⼀个dataavailable的Event，在这⾥做视频数据的采集⼯作，主要是基于Blob进⾏转写，利⽤FileReader进⾏读取。FileReader⼀定
//要注册loadend的，或者写onload的函数。在loadend的监听函数⾥⾯，进⾏格式转换，⽅便websocket进⾏数据传输，因为websocket的数据类型⽀持blob以及arrayBuffer，我们这⾥⽤
//的是arrayBuffer，所以，将视频数据的Blob转写为Unit8Buffer，便于websocket的后台服务⽤ByteBuffer接收。
//log('');
//console.log(e.data);
//console.log(pe);
//console.log(e);
chunks.push(e.data);
var reader = new FileReader();
reader.addEventListener("loadend", function() {
//sult是⼀个含有视频数据流的Blob对象
var buf = new sult);
console.sult);
sult.byteLength > 0){        //加这个判断，是因为有很多数据是空的，这个没有必要发到后台服务器，减轻⽹络开销，提升性能吧。
ws.send(buf);
}
});
};
log('Error: ' + e);
};
log('Started & state = ' + mediaRecorder.state);
};
log('Stopped  & state = ' + mediaRecorder.state);
var blob = new Blob(chunks, {type: "video/webm"});
chunks = [];
var videoURL = ateObjectURL(blob);
downloadLink.href = videoURL;
videoElement.src = videoURL;
downloadLink.innerHTML = 'Download video file';
var rand =  Math.floor((Math.random() * 10000000));
var name  = "video_"+rand+".webm" ;
downloadLink.setAttribute( "download", name);
downloadLink.setAttribute( "name", name);
};
log('Paused & state = ' + mediaRecorder.state);
}
log('Resumed  & state = ' + mediaRecorder.state);
}
log('Warning: ' + e);
};
}
//function handleSourceOpen(event) {
//  console.log('MediaSource opened');
//  sourceBuffer = mediaSource.addSourceBuffer('video/webm; codecs="vp9"');
//  console.log('Source buffer: ', sourceBuffer);
//}
//点击按钮，启动视频流的采集。重点是getUserMedia函数使⽤。本案例中，视频采集的⼊⼝，是点击页⾯上的record按钮，也就是下⾯这个函数的逻辑。function onBtnRecordClicked (){
if (typeof MediaRecorder === 'undefined' || !UserMedia) {
alert('MediaRecorder not supported on your browser, use Firefox 30 or Chrome 49 instead.');
}else {
recBtn.disabled = true;
pauseResBtn.disabled = false;
stopBtn.disabled = false;
}
}
function onBtnStopClicked(){
mediaRecorder.stop();
recBtn.disabled = false;
pauseResBtn.disabled = true;
stopBtn.disabled = true;
}
function onPauseResumeClicked(){
Content === "Pause"){
console.log("pause");
mediaRecorder.pause();
stopBtn.disabled = true;
}else{
console.log("resume");
stopBtn.disabled = false;
}
recBtn.disabled = true;
pauseResBtn.disabled = false;
}
function log(message){
dataElement.innerHTML = dataElement.innerHTML+'<br>'+message ;
}
//browser ID
function getBrowser(){
var nVer = navigator.appVersion;
var nAgt = navigator.userAgent;
var browserName  = navigator.appName;
var fullVersion  = ''+parseFloat(navigator.appVersion);
var majorVersion = parseInt(navigator.appVersion,10);
var nameOffset,verOffset,ix;
// In Opera, the true version is after "Opera" or after "Version"
if ((verOffset=nAgt.indexOf("Opera"))!=-1) {
browserName = "Opera";
fullVersion = nAgt.substring(verOffset+6);
if ((verOffset=nAgt.indexOf("Version"))!=-1)
fullVersion = nAgt.substring(verOffset+8);
}
// In MSIE, the true version is after "MSIE" in userAgent
else if ((verOffset=nAgt.indexOf("MSIE"))!=-1) {
browserName = "Microsoft Internet Explorer";
fullVersion = nAgt.substring(verOffset+5);
}
// In Chrome, the true version is after "Chrome"
else if ((verOffset=nAgt.indexOf("Chrome"))!=-1) {
browserName = "Chrome";
fullVersion = nAgt.substring(verOffset+7);
}
// In Safari, the true version is after "Safari" or after "Version"
else if ((verOffset=nAgt.indexOf("Safari"))!=-1) {
browserName = "Safari";
fullVersion = nAgt.substring(verOffset+7);
if ((verOffset=nAgt.indexOf("Version"))!=-1)
fullVersion = nAgt.substring(verOffset+8);
}
// In Firefox, the true version is after "Firefox"
else if ((verOffset=nAgt.indexOf("Firefox"))!=-1) {
browserName = "Firefox";
fullVersion = nAgt.substring(verOffset+8);
}
// In most other browsers, "name/version" is at the end of userAgent
else if ( (nameOffset=nAgt.lastIndexOf('')+1) <
(verOffset=nAgt.lastIndexOf('/')) )html实现用户注册登录代码
{
browserName = nAgt.substring(nameOffset,verOffset);
fullVersion = nAgt.substring(verOffset+1);
if (LowerCase()==UpperCase()) {
browserName = navigator.appName;
}
}
// trim the fullVersion string at semicolon/space if present
if ((ix=fullVersion.indexOf(";"))!=-1)
fullVersion=fullVersion.substring(0,ix);
if ((ix=fullVersion.indexOf(""))!=-1)
fullVersion=fullVersion.substring(0,ix);
majorVersion = parseInt(''+fullVersion,10);
if (isNaN(majorVersion)) {
fullVersion  = ''+parseFloat(navigator.appVersion);
majorVersion = parseInt(navigator.appVersion,10);
}
return browserName;
}
其中的byteLength的判断，是有原因的，前端打印的⽇志可以看出：
我的这个案例，⽤的是Firefox的浏览器，因为我本地的Chrome的版本⽐较新，在应⽤启动的时候爆出错误：
时间紧，没有深⼊研究这个错误，所以⼀直都是Firefox基础上进⾏验证的。
下⾯剩下的就是Java后台的Websocket的服务了。直接上代码：
/*
* Copyright © reserved by roomdis, service for tgn company whose important business is rural e-commerce.
*/
dis.;
import java.io.IOException;
import java.DatagramPacket;
import java.DatagramSocket;
import java.InetAddress;
import java.SocketException;
import java.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Map;
import urrent.ConcurrentHashMap;
import javax.websocket.OnClose;
import javax.websocket.OnError;
import javax.websocket.OnMessage;
import javax.websocket.OnOpen;
import javax.websocket.Session;
import javax.websocket.server.ServerEndpoint;
import org.apache.log4j.Logger;
import org.t.ContextLoader;
le.gson.Gson;
dis.mqr.infra.msg.KefuMessage;
/**
* @author shihuc
* @date 2017年8⽉22⽇下午2:20:18
*/
@ServerEndpoint("/websocket")
public class WebsocketService {
private static Logger logger = Logger(WebsocketService.class);
private HttpSendService httpSendService;
private String videoRecServerHost = "10.90.7.10";
private int videoRecServerPort = 7667;
/*
* 当存在多个客户端访问时，为了保证会话继续保持，将连接缓存。
*/
private static Map<String, WebsocketService> webSocketMap = new ConcurrentHashMap<String, WebsocketService>();
private Session session;
private static final WebsocketService instance = new WebsocketService();
public static final WebsocketService getInstance() {
return instance;
}
@OnMessage
public void onTextMessage(String message, Session session) throws IOException, InterruptedException {
// Print the client message for testing purposes
logger.info("Received: " + message);
//TODO: 调⽤接⼝将消息发送给客户端后台服务系统
Gson gson = new Gson();
KefuMessage kfMsg = gson.fromJson(message, KefuMessage.class);
httpSendService = CurrentWebApplicationContext().getBean(HttpSendService.class);
}
/**
* 主要⽤来接受⼆进制数据。
*
* @author shihuc
* @param message
* @param session
* @throws IOException
* @throws InterruptedException
*/
@OnMessage
public void onBinaryMessage(ByteBuffer message, Session session, boolean last) throws IOException, InterruptedException {        byte [] sentBuf = message.array();
logger.info("Binary Received: " + sentBuf.length + ", last: " + last);
//下⾯的代码逻辑，是⽤UDP协议发送视频流数据到视频处理服务器做后续逻辑处理
//sendToVideoRecognizer(sentBuf);
}
/**
* @author shihuc
* @param sentBuf
* @throws SocketException
* @throws UnknownHostException
* @throws IOException
*/
private void sendToVideoRecognizer(byte[] sentBuf) throws SocketException, UnknownHostException, IOException {
DatagramSocket client = new DatagramSocket();
InetAddress addr = ByName(videoRecServerHost);
DatagramPacket sendPacket = new DatagramPacket(sentBuf, sentBuf.length, addr, videoRecServerPort);
client.send(sendPacket);
client.close();
}
//    @OnOpen
//    public void onOpen(Session session){
//        this.session = session;
//        String staffId = QueryString();
//        webSocketMap.put(staffId, this);
//        logger.info(staffId + " client opened");
//    }
@OnOpen
public void onOpen(Session session){
logger.info("client opened: " + String());
}
@OnClose
public void onClose() {
logger.info("client onclose");
}
@OnError
public void onError(Session session, Throwable error){
logger.info("connection onError");
logger.Cause());
}
public boolean sendMessage(String message, String staffId) throws IOException{
WebsocketService client = (staffId);
if (client == null) {
return false;
}
boolean result=false;
try {
BasicRemote().sendText(message);
result=true;
} catch (IOException e) {
try {
client.session.close();
} catch (IOException e1) {
e1.printStackTrace();
}
}
return result;
}
}
这⾥，重点要注意的是，@OnMessage注解对应的函数，⼊参⾮常有讲究的。对于arrayBuffer的⼆进制数据类型，参数个数必须是三个，最后的boolean的必须有，否则前端发送数据的时候，浏览器上会抛出错误：
最后，看看后台运⾏的⽇志：
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 8192, last: false
[2017-09-2720:03:47] [ INFO] [http-nio-9080-exec-7] [dis.BinaryMessage(WebsocketService.java:80)] - Binary Received: 7494, last: true
并附上⼀副前端运⾏的效果截图：
总结：
1. 重点研究getUserMedia。
2.重点研究MediaRecorder。
3.重点研究Blob以及FileReader。
4.重点研究Websocket的@OnMessage的注解函数的参数，以及数据传输中连接可能会断掉的处理⽅案。
2018-05-03
希望有兴趣的朋友，通过关注我的博客，共同互动，研究⼀些特别的应⽤！