網上很多關于MSE相關文章都有講過有些注意點講的并不清楚，比如接收的視訊資料格式、flash相容性問題、imie格式不對、無法做到實時接收資料、重新整理後後續資料無法播放等等
現在整體講一下
該工程通過浏覽器頁面調用錄屏功能将視訊資料傳到後端，後端通過websocket發送資料到第三方頁面進行實時播放
之前嘗試過rtmp+ffmpeg+nginx進行推流直播，但是發現ffmpeg需要讀取完整視訊檔案才行和需求不通
環境：chrome 92.0.4515.131
jdk:1.8
視訊資料采集
record.html
<html>
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Screen Record</title>
</head>
<style>
video {
border: 1px solid #999;
width: 98%;
max-width: 860px;
}
</style>
<body>
<p>This example shows you the contents of the selected part of your display.
Click the Start Capture button to begin.</p>
<p><button id="start">Start Capture</button> <button id="stop">Stop Capture</button></p>
<video id="video" autoplay></video>
</body>
<script>
const videoElem = document.getElementById("video");
const startElem = document.getElementById("start");
const stopElem = document.getElementById("stop");
// Options for getDisplayMedia()
const displayMediaOptions = {
video: {
cursor: "always"
,frameRate: "8"
,height:500
},
audio: false
};
// Set event listeners for the start and stop buttons
startElem.addEventListener("click", function(evt) {
startCapture();
}, false);
stopElem.addEventListener("click", function(evt) {
stopCapture();
}, false);
let recorder;
var stream;
async function startCapture() {
try {
stream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions);
videoElem.srcObject = stream;
var options = {
mimeType: 'video/webm;codecs=vp8'
}
recorder = new MediaRecorder(stream, options);
recorder.ondataavailable = e => {
put(e.data);
};
recorder.start(2000);
} catch(err) {
console.error("Error: " + err);
}
}
function stopCapture(evt) {
//recorder.stop();
let tracks = videoElem.srcObject.getTracks();
tracks.forEach(track => track.stop());
videoElem.srcObject = null;
}
function put(d){
var aa = [];
aa.push(d);
var formData = new FormData();
formData.append("user", "aaa");
// JavaScript file-like 對象
let nd = new Blob(aa);
formData.append("file", nd);
var request = new XMLHttpRequest();
request.open("POST", "/jsmp4");
request.send(formData);
}
</script>
</html>
注意：該視訊格式為webm格式，編碼方式為vp8
後端接收處理
package com.r.controller;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import com.r.common.vo.common.RespResult;
import com.r.controller.common.BaseController;
import com.r.util.FuncUtil;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@RestController
public class Test {
@RequestMapping("/jsmp4")
@ResponseBody
public RespResult uploadMp4File(MultipartFile file, String user) {
if(FuncUtil.isNull(user)) {
return new RespResult(RESULT_ERROR, "參數錯誤");
}
//儲存臨時檔案
int len = 0;
try {
len = file.getBytes().length;
} catch (Exception e) {
}
log.info("file.name==" + user + ", length=" + len);
File temp = new File("E:\\ftp\\tmp\\"+user +".mp4");
FileOutputStream fos = null;
try {
fos = new FileOutputStream(temp,true);
fos.write(file.getBytes());
fos.flush();
fos.close();
} catch (FileNotFoundException e) {
log.error("error:",e);
} catch (IOException e) {
log.error("error:",e);
} finally {
if(fos!=null) {
try {
fos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
try {
//通過websocket推送資料
RedisWebSocket.sendMassage(file.getBytes());
} catch (Exception e) {
e.printStackTrace();
}
return new RespResult(RESULT_SUCCESS, "成功");
}
}
websocket推送資料
package com.r.controller;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import javax.websocket.OnClose;
import javax.websocket.OnError;
import javax.websocket.OnMessage;
import javax.websocket.OnOpen;
import javax.websocket.Session;
import javax.websocket.server.ServerEndpoint;
import org.springframework.stereotype.Component;
import com.r.util.WebmUtils
import lombok.extern.slf4j.Slf4j;
@Slf4j
@Component
@ServerEndpoint("/ws/redis")
public class RedisWebSocket {
//sessionid 為Key作為使用者辨別
public static ConcurrentHashMap<String,Session> sessionMap = new ConcurrentHashMap<String,Session>();
public static ConcurrentHashMap<String,String> sessionData = new ConcurrentHashMap<String,String>();
/**
* 新的WebSocket請求開啟
*/
@OnOpen
public void onOpen(Session session) {
log.info("ws redis-------open, key=" + session.getId());
sessionMap.put(session.getId(), session);
}
public static void sendMassage(String text) {
try {
for (String key : sessionMap.keySet()) {
//log.info(text);
if(key == null) {
sessionMap.remove(key);
continue;
}
Session ses = sessionMap.get(key);
if(ses == null || !ses.isOpen()) {
sessionMap.remove(key);
continue;
}
try {
// 将實時日志通過WebSocket發送給用戶端，給每一行添加一個HTML換行
ses.getBasicRemote().sendText(text);
} catch (IOException e) {
log.error("websocket error:", e);
}
}
} catch (Exception e) {
log.error("websocket error:", e);
}
}
public static void sendMassage(byte[] data) {
try {
for (String key : sessionMap.keySet()) {
//log.info(text);
if(key == null) {
sessionMap.remove(key);
continue;
}
Session ses = sessionMap.get(key);
if(ses == null || !ses.isOpen()) {
sessionMap.remove(key);
continue;
}
try {
//有資料，加頭
if(sessionData.containsKey(key)) {
byte[] new_data = headCheck(data);
if(new_data == null) {
System.out.println("add head is null");
continue;
}
System.out.println("add head data:" + data.length + ", new_data:" + new_data.length);
data = new_data;
sessionData.remove(key);
}
ses.getBasicRemote().sendBinary(ByteBuffer.wrap(data));
} catch (Exception e) {
log.error("websocket error:", e);
}
}
} catch (Exception e) {
log.error("websocket error:", e);
}
}
/**
* WebSocket請求關閉
*/
@OnClose
public void onClose(Session session) {
sessionMap.remove(session.getId());
log.info("ws redis-------close");
}
@OnError
public void onError(Session session, Throwable thr) {
log.error("websocket error:", thr);
log.info("ws redis-------error key="+(session!=null ? session.getId() : ""));
}
/**
* 擷取消息響應
* @param session 可選參數
* @param message 接收到的消息
*/
@OnMessage
public void onMessage(Session session, String message) {
log.info("session id:"+ session.getId()+", onmessage=" + message + ", session.requestPara=" + session.getRequestParameterMap());
if(sessionMap.containsKey(session.getId())) {
sessionData.put(session.getId(), message);
}
}
}
後端接收頁面發送的資料來判斷是否添加視訊頭，使得頁面可以重新播放後續資料。
裡面用到的headCheck方法将在下面介紹
頁面實時播放
stream.html
<html>
<meta charset="utf-8">
<head>
<style type="text/css">
#video {
border: 1px solid #999;
width: 98%;
max-width: 860px;
}
</style>
</head>
<body>
<video id="video" controls muted autoplay>
</video>
<script>
var mimeCodec = 'video/webm;codecs=vp8';
var video = document.querySelector('#video');
if ('MediaSource' in window && MediaSource.isTypeSupported(mimeCodec)) {
var mediaSource = new MediaSource;
//console.log(mediaSource.readyState); // closed
video.src = URL.createObjectURL(mediaSource);
mediaSource.addEventListener('sourceopen', sourceOpen);
} else {
console.error('Unsupported MIME type or codec: ', mimeCodec);
}
function sourceOpen (e) {
URL.revokeObjectURL(video.src)
//console.log(this.readyState); // open
var mediaSource = e.target;
var sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
sourceBuffer.mode = 'sequence';
sourceBuffer.addEventListener('updateend', function (_) {
mediaSource.endOfStream();
video.play();
//console.log(mediaSource.readyState); // ended
});
var websocket = new WebSocket('ws://127.0.0.1:36886/ws');
websocket.binaryType = 'arraybuffer';
websocket.onopen = function (){
websocket.send("add head");
}
websocket.onmessage = function(message) {
var data = message.data;
//var ct = video.currentTime ;
sourceBuffer.appendBuffer(data);
//video.currentTime = ct;
};
websocket.onclose = function (event) {
console.log("close");
}
};
</script>
</body>
</html>
注意：MediaSource接收的視訊格式必須和mime聲明的一樣，支援fmp4（Fragmented mp4），普通mp4是不行的，錄屏資料本身就是他們生成的也是支援的
我的資料是webm的編碼方式為vp8，沒有音頻是以寫成 "video/webm;codecs=vp8"
如果是fmp4的需要寫成'video/mp4; codecs="mp4a.40.2,avc1.64001f"'，其中avc1.64001f是音頻編碼，根據時間情況修改mime類型。可通過MediaSource.isTypeSupported(mimeCodec)檢查浏覽器是否支援該類型。
fmp4需要有flash插件支援
參考：H5直播系列二 MSE(Media Source Extensions)
資料流解析
webm遵循EBML規範（Extended Binary Markup Language）
參考：
【多媒體封裝格式詳解】---MKV【1】
【多媒體封裝格式詳解】---MKV【2】
【多媒體封裝格式詳解】---MKV【3】完
官方網站：https://www.matroska.org/technical/elements.html
資料格式是這樣的：mkvtoolnix工具可以檢視
視訊開頭包含前122個位元組
後續發送的資料都沒有是簇 資料，并且資料開始的幀并不完整。工具無法正常打開，因為并不是完整視訊檔案
開頭122個位元組資料基本固定，是以我就直接吧它拼接到後續傳過來的資料中寫成檔案，浏覽器可以正常打開播放，但是用MSE時間上打不開，浏覽器實際上有一定的容錯性。
将後續不完整資料進行檢查，逐個自己檢查是否包含“簇”的ID（0x1F43B675），找到後在檢查後續自己是否符合格式，不符合繼續檢查，符合加上開頭122個位元組，可以正常播放。
嘗試過檢查簡單塊的ID（0xA3）相當于一個幀，0xA3的位元組後續資料也符合格式将該幀之前不完整的資料去掉，然後再前面加上固定的開頭122個位元組和補齊簇相關代碼。發現視訊還是無法播放，應該是和幀數不足以及簇時間戳不對有關，隻能放棄這個想法。
com.r.util.WebmUtils
package com.r.util;
import com.r.common.KeyValuePair;
import com.r.util.AESUtil;
public class WebmUtils {
private static String webm_vp8_head = "1A45DFA39F4286810142F7810142F2810442F381084282847765626D42878104428581021853806701FFFFFFFFFFFFFF1549"
+ "A966992AD7B1830F42404D80864368726F6D655741864368726F6D651654AE6BABAEA9D7810173C5874F0AABE521FEC58381"
+ "0155EE81018685565F565038E08CB0820600BA82036053C08101";
private static byte[] webm_vp8_head_byte = AESUtil.parseHexStr2Byte(webm_vp8_head);
public static byte[] headCheck(byte[] data) {
KeyValuePair<String, Integer> kv = getEbmlIndex(data);
int h_len = webm_vp8_head_byte.length;
int d_len = data.length - kv.getValue();
byte[] new_data = null;
if(kv.getKey().equals("0x1F43B675")) {
new_data = new byte[h_len + d_len];
System.arraycopy(webm_vp8_head_byte, 0, new_data, 0, h_len);
System.arraycopy(data, kv.getValue(), new_data, h_len, d_len);
} else if(kv.getKey().equals("0xA3")) {
String ebml_cluster = "1F43B67501FFFFFFFFFFFFFFE78100";//時間戳從0開始
byte[] ebml_cluster_byte = AESUtil.parseHexStr2Byte(ebml_cluster);
int cl_len = ebml_cluster_byte.length;
new_data = new byte[h_len + cl_len + d_len];
System.arraycopy(webm_vp8_head_byte, 0, new_data, 0, h_len);
System.arraycopy(ebml_cluster_byte, 0, new_data, h_len, cl_len);
System.arraycopy(data, kv.getValue(), new_data, h_len + cl_len, d_len);
//new_data[h_len + cl_len + 6] = Integer.valueOf("10000000", 2).byteValue();//改成關鍵幀?
}
return new_data;
}
/**
* 找0xA3和0x1F43B675
* 0xA3幀數不足無法播放，不在找
* @param data
* @return
*/
private static KeyValuePair<String, Integer> getEbmlIndex(byte[] data) {
long begin = System.currentTimeMillis();
long totallength = data.length;
System.out.println("資料總長度: " + data.length);
String ebml = "";
int offset = 0;
int index = 0;
for (; index < totallength; index++) {
offset = index;
/*if((data[offset] & 0xFF) == 0xA3) {
System.out.println();
System.out.println("0xA3 發現 offset: " + offset);
if(offset + 1 >= totallength) {
System.out.println("0xA3 長度讀取為-1, 結束");
break;
}
offset++;
byte a3_len_1byte = data[offset];
if((a3_len_1byte & 0xFF) == 0) {
System.out.println("0xA3 長度是0, 重新找, offset: " + offset);
continue;
}
String a3_len_1byte_bin = Integer.toBinaryString(a3_len_1byte & 0xFF);
System.out.println("0xA3 長度首位元組二進制數: " + a3_len_1byte_bin);
int a3_len_1byte_after = 8 - a3_len_1byte_bin.length();
System.out.println("0xA3 長度位元組數: "+ (a3_len_1byte_after + 1));
if(offset + a3_len_1byte_after >= totallength) {
System.out.println("0xA3 長度後續讀取為不足-1, 重新找, offset: " + offset);
continue;
}
byte[] a3_len_byte = new byte[a3_len_1byte_after + 1];
a3_len_byte[0] = a3_len_1byte;
for (int j = 1; j < a3_len_byte.length; j++) {
offset++;
a3_len_byte[j] = data[offset];
}
System.out.println("0xA3 長度十六進制: "+ AESUtil.parseByte2HexStr(a3_len_byte));
long a3_len_val = ebmlLen(a3_len_byte);
System.out.println("0xA3 長度去符号後的值: " + a3_len_val);
if(offset + a3_len_val >= totallength) {
System.out.println("0xA3 長度 " + a3_len_val + " 大于剩餘 " + (totallength - offset) + ", 重新找, offset: " + offset);
continue;
}
//跳過資料
offset += a3_len_val;
//0xA3後續讀取0x1F43B675或0xA3
//0x1F43B675 0xA3
byte[] a3Or1f = new byte[4];
if(offset + 1 >= totallength) {
System.out.println("0xA3 後續讀取 0xA3 or 0x1F43B675 讀取為-1, 重新找, offset: " + offset);
continue;
}
offset++;
a3Or1f[0] = data[offset];
int a3_0 = a3Or1f[0] & 0xFF;
if(a3_0 != 0xA3 && a3_0 != 0x1F) {
System.out.println("0xA3 後續讀 0x1F43B675 or 0xA3 沒有發現: " + Integer.toHexString(a3_0) + ", 重新找, offset: " + offset);
continue;
}
if(a3_0 == 0xA3) {
System.out.println("0xA3 後續讀 0xA3 找到: 位置為: " + offset);
ebml="0xA3";
break;
}
//s 368821 o 5885
if(offset + 3 >= totallength) {
System.out.println("0xA3 後續讀取 0x1F43B675 讀取為-1, 重新找, offset: " + offset);
continue;
}
for (int j = 1; j < a3Or1f.length; j++) {
offset++;
a3Or1f[j] = data[offset];
}
String a3_1f43b675_hex = AESUtil.parseByte2HexStr(a3Or1f);
if("1F43B675".equalsIgnoreCase(a3_1f43b675_hex)) {
System.out.println("0xA3 後續讀 0x1F43B675 找到, 位置為: " + (offset-3));
ebml = "0x1F43B675";
break;
} else {
System.out.println("0xA3 後續讀 0x1F43B675沒有發現: " + a3_1f43b675_hex + ", 重新找, offset: " + offset);
continue;
}
} else */ if((data[offset] & 0xFF) == 0x1F){
System.out.println();
System.out.println("0x1F43B675 發現0x1F offset: "+offset);
if(offset + 3 >= totallength) {
System.out.println("0x1F43B675 後續讀取 0x1F43B675 讀取為-1, 結束");
break;
}
byte[] x1f43b675 = new byte[4];
x1f43b675[0] = data[offset];
for (int j = 1; j < x1f43b675.length; j++) {
offset++;
x1f43b675[j] = data[offset];
}
String x1f43b675_hex = AESUtil.parseByte2HexStr(x1f43b675);
if(!"1F43B675".equalsIgnoreCase(x1f43b675_hex)) {
System.out.println("0x1F43B675 後續錯誤: " + x1f43b675_hex + ", 重新找, offset: " + offset);
continue;
}
System.out.println("0x1F43B675 找到, 位置為: " + offset + ", 檢查後續長度");
if(offset + 1 >= totallength) {
System.out.println("0x1F43B675 長度讀取為-1, 重新找, offset: " + offset);
continue;
}
offset++;
if((data[offset] & 0xFF) == 0) {
System.out.println("0x1F43B675 長度是0, 重新找, offset: " + offset);
continue;
}
String x1f43b675_len_1byte = Integer.toBinaryString(data[offset] & 0xFF);
System.out.println("0x1F43B675 長度首位元組二進制數: " + x1f43b675_len_1byte);
int x1f43b675_len_1byte_after = 8 - x1f43b675_len_1byte.length();
System.out.println("0x1F43B675 長度位元組數: "+ (x1f43b675_len_1byte_after + 1));
//不檢查簇長度、未知的
if(offset + x1f43b675_len_1byte_after >= totallength) {
System.out.println("0x1F43B675 跳過長度失敗: " + x1f43b675_len_1byte_after + ", 結束");
continue;
}
offset+=x1f43b675_len_1byte_after;
//檢查資料
if(offset + 1 >= totallength) {
System.out.println("0x1F43B675 找0xE7讀取為-1, 重新找, offset: " + offset);
continue;
}
offset++;
if((data[offset] & 0xFF) != 0xE7) {
System.out.println("0x1F43B675 0xE7 沒有找到 , 重新找, offset: " + offset);
continue;
}
if(offset + 1 >= totallength) {
System.out.println("0x1F43B675 0xE7 長度讀取為-1, 重新找, offset: " + offset);
continue;
}
offset++;
byte e7_len_1byte = data[offset];
if((e7_len_1byte & 0xFF) == 0) {
System.out.println("0x1F43B675 0xE7長度是0, 重新找, offset: " + offset);
continue;
}
String e7_len_1byte_bin = Integer.toBinaryString(e7_len_1byte & 0xFF);
System.out.println("0x1F43B675 0xE7長度首位元組二進制數: " + e7_len_1byte_bin);
int e7_len_1byte_after = 8 - e7_len_1byte_bin.length();
System.out.println("0x1F43B675 0xE7長度位元組數: "+ (e7_len_1byte_after + 1));
if(offset + e7_len_1byte_after >= totallength) {
System.out.println("0x1F43B675 0xE7長度讀取不足, 重新找, offset: " + offset);
continue;
}
byte[] e7_len_byte = new byte[e7_len_1byte_after + 1];
e7_len_byte[0] = e7_len_1byte;
for (int j = 1; j < e7_len_byte.length; j++) {
offset++;
e7_len_byte[j] = data[offset];
}
System.out.println("0x1F43B675 0xE7長度十六進制: " + AESUtil.parseByte2HexStr(e7_len_byte));
long a7_len_val = ebmlLen(e7_len_byte);
System.out.println("0x1F43B675 0xE7 長度去符号後的值: " + a7_len_val);
if(offset + a7_len_val >= totallength) {
System.out.println("0x1F43B675 0xE7 長度" + a7_len_val + " 大于剩餘 " + (totallength - offset) + ", 重新找, offset: " + offset);
continue;
}
//跳過資料
offset += a7_len_val;
//讀取0x1F43B675後0xA3
if(offset + 1 >= totallength) {
System.out.println("0x1F43B675 後續讀取0xA3 讀取為-1, 重新找, offset: " + offset);
continue;
}
offset++;
byte x1F43B675_a3 = data[offset];
if((x1F43B675_a3 & 0xFF) == 0xA3) {
System.out.println("0x1F43B675 0xA3 找到，位置為:" + offset);
ebml = "0x1F43B675";
break;
} else {
String x1F43B675_a3_hex = Integer.toHexString(x1F43B675_a3 & 0xFF);
System.out.println("0x1F43B675 0xA3 沒有找到: " + x1F43B675_a3_hex + ", 重新找, offset: " + offset);
continue;
}
}
}
long end = System.currentTimeMillis();
System.out.println(ebml + " index: " + index + " hex: " + Long.toHexString(index) + ", time: " + (end-begin) +"ms");
return new KeyValuePair<String, Integer>(ebml, index);
}
private static long ebmlLen(byte[] lens) {
String len0 = Integer.toBinaryString(lens[0] & 0xFF);
long val = 0;
if(!len0.equals("1")) {
int tmp = Integer.valueOf(len0.substring(1), 2);
val = tmp;
}
for (int i = 1; i < lens.length; i++) {
val = (val << 8) + (lens[i] & 0xFF);
}
return val;
}
該方法針對chrom産生的webm視訊資料有效，其他視訊格式自行修改
com.r.util.AESUtil.java
public static String parseByte2HexStr(byte buf[]) {
StringBuffer sb = new StringBuffer();
for (int i = 0; i < buf.length; i++){
String hex = Integer.toHexString(buf[i] & 0xFF);
if (hex.length() == 1){
hex = '0' + hex;
}
sb.append(hex.toUpperCase());
}
return sb.toString();
}
/**
* Desc:将16進制轉換為二進制 param:String hexStr return:byte[] Author:WangPeiqiang
* Date:2017/8/22
**/
public static byte[] parseHexStr2Byte(String hexStr) {
if (hexStr.length() < 1)
return null;
byte[] result = new byte[hexStr.length() / 2];
for (int i = 0; i < hexStr.length() / 2; i++){
int high = Integer.parseInt(hexStr.substring(i * 2, i * 2 + 1), 16);
int low = Integer.parseInt(hexStr.substring(i * 2 + 1, i * 2 + 2), 16);
result[i] = (byte) (high * 16 + low);
}
return result;
}
初次接觸視訊流處理方面的相關知識，有錯誤的地方請指正。