I am trying to record RTP stream using FFmpeg. but i can not make it work on Firefox. It sounds like RTP packets are not sending when i am using Firefox. However chrome works fine.
Client side code is a vue js component which gets rtp compabilities using socket and create producers after the server crearted the transport
<template>
<v-row no-gutters justify="center" ref="videoContainer" class="mt-2">
<div class="video-recorder_wrapper">
<video v-if="recordingMode ==='sapio'"
class="video-camera rounded" ref="video" style="height: 300px">
</video>
<div v-if="isRecording" class="pa-1" style="position: absolute; bottom:5px; left:5px; margin:3px;">
<v-chip color="white"
text-color="red"
small
>
<div class="pulse-circle mr-1" ></div>
recording
</v-chip>
</div>
</div>
</v-row>
<v-row v-if="Q.options.showControl" justify="center" no-gutters class="my-2">
<v-btn @click="stopRecording"
v-if="isRecording"
color="primary"
outlined
rounded
>
<v-icon>mdi-stop</v-icon>
stop
</v-btn>
<v-btn @click="startRecording"
v-if="!isRecording"
:loading="loading"
color="red"
outlined
rounded
>
<v-icon>mdi-record</v-icon>
record
</v-btn>
</v-row>
</template>
<script>
const { connect , createLocalTracks } = require('twilio-video');
const SocketClient = require("socket.io-client");
const SocketPromise = require("socket.io-promise").default;
const MediasoupClient = require("mediasoup-client");
export default {
data() {
return {
errors: [],
isReady: false,
isRecording: false,
loading: false,
sapio: {
token: null,
connectionId: 0
},
server: {
host: 'https://rtc.test',
ws: '/server',
socket: null,
},
peer: {},
}
},
mounted() {
this.init();
},
methods: {
async init() {
await this.startCamera();
if (this.takeId) {
await this.recordBySapioServer();
}
},
startCamera() {
return new Promise( (resolve, reject) => {
if (window.videoMediaStreamObject) {
this.setVideoElementStream(window.videoMediaStreamObject);
resolve();
} else {
// Get user media as required
try {
this.localeStream = navigator.mediaDevices.getUserMedia({
audio: true,
video: true,
}).then((stream) => {
this.setVideoElementStream(stream);
resolve();
})
} catch (err) {
console.error(err);
reject();
}
}
})
},
setVideoElementStream(stream) {
this.localStream = stream;
this.$refs.video.srcObject = stream;
this.$refs.video.muted = true;
this.$refs.video.play().then((video) => {
this.isStreaming = true;
this.height = this.$refs.video.videoHeight;
this.width = this.$refs.video.videoWidth;
});
},
// first thing we need is connecting to websocket
connectToSocket() {
const serverUrl = this.server.host;
console.log("Connect with sapio rtc server:", serverUrl);
const socket = SocketClient(serverUrl, {
path: this.server.ws,
transports: ["websocket"],
});
this.socket = socket;
socket.on("connect", () => {
console.log("WebSocket connected");
// we ask for rtp-capabilities from server to send to us
socket.emit('send-rtp-capabilities');
});
socket.on("error", (err) => {
this.loading = true;
console.error("WebSocket error:", err);
});
socket.on("router-rtp-capabilities", async (msg) => {
const { routerRtpCapabilities, sessionId, externalId } = msg;
console.log('[rtpCapabilities:%o]', routerRtpCapabilities);
this.routerRtpCapabilities = routerRtpCapabilities;
try {
const device = new MediasoupClient.Device();
// Load the mediasoup device with the router rtp capabilities gotten from the server
await device.load({ routerRtpCapabilities });
this.peer.sessionId = sessionId;
this.peer.externalId = externalId;
this.peer.device = device;
this.createTransport();
} catch (error) {
console.error('failed to init device [error:%o]', error);
socket.disconnect();
}
});
socket.on("create-transport", async (msg) => {
console.log('handleCreateTransportRequest() [data:%o]', msg);
try {
// Create the local mediasoup send transport
this.peer.sendTransport = await this.peer.device.createSendTransport(msg);
console.log('send transport created [id:%s]', this.peer.sendTransport.id);
// Set the transport listeners and get the users media stream
this.handleSendTransportListeners();
this.setTracks();
this.loading = false;
} catch (error) {
console.error('failed to create transport [error:%o]', error);
socket.disconnect();
}
});
socket.on("connect-transport", async (msg) => {
console.log('handleTransportConnectRequest()');
try {
const action = this.connectTransport;
if (!action) {
throw new Error('transport-connect action was not found');
}
await action(msg);
} catch (error) {
console.error('ailed [error:%o]', error);
}
});
socket.on("produce", async (msg) => {
console.log('handleProduceRequest()');
try {
if (!this.produce) {
throw new Error('produce action was not found');
}
await this.produce(msg);
} catch (error) {
console.error('failed [error:%o]', error);
}
});
socket.on("recording", async (msg) => {
this.isRecording = true;
});
socket.on("recording-error", async (msg) => {
this.isRecording = false;
console.error(msg);
});
socket.on("recording-closed", async (msg) => {
this.isRecording = false;
console.warn(msg)
});
},
createTransport() {
console.log('createTransport()');
if (!this.peer || !this.peer.device.loaded) {
throw new Error('Peer or device is not initialized');
}
// First we must create the mediasoup transport on the server side
this.socket.emit('create-transport',{
sessionId: this.peer.sessionId
});
},
handleSendTransportListeners() {
this.peer.sendTransport.on('connect', this.handleTransportConnectEvent);
this.peer.sendTransport.on('produce', this.handleTransportProduceEvent);
this.peer.sendTransport.on('connectionstatechange', connectionState => {
console.log('send transport connection state change [state:%s]', connectionState);
});
},
handleTransportConnectEvent({ dtlsParameters }, callback, errback) {
console.log('handleTransportConnectEvent()');
try {
this.connectTransport = (msg) => {
console.log('connect-transport action');
callback();
this.connectTransport = null;
};
this.socket.emit('connect-transport',{
sessionId: this.peer.sessionId,
transportId: this.peer.sendTransport.id,
dtlsParameters
});
} catch (error) {
console.error('handleTransportConnectEvent() failed [error:%o]', error);
errback(error);
}
},
handleTransportProduceEvent({ kind, rtpParameters }, callback, errback) {
console.log('handleTransportProduceEvent()');
try {
this.produce = jsonMessage => {
console.log('handleTransportProduceEvent callback [data:%o]', jsonMessage);
callback({ id: jsonMessage.id });
this.produce = null;
};
this.socket.emit('produce', {
sessionId: this.peer.sessionId,
transportId: this.peer.sendTransport.id,
kind,
rtpParameters
});
} catch (error) {
console.error('handleTransportProduceEvent() failed [error:%o]', error);
errback(error);
}
},
async recordBySapioServer() {
this.loading = true;
this.connectToSocket();
},
async setTracks() {
// Start mediasoup-client's WebRTC producers
const audioTrack = this.localStream.getAudioTracks()[0];
this.peer.audioProducer = await this.peer.sendTransport.produce({
track: audioTrack,
codecOptions :
{
opusStereo : 1,
opusDtx : 1
}
});
let encodings;
let codec;
const codecOptions = {videoGoogleStartBitrate : 1000};
codec = this.peer.device.rtpCapabilities.codecs.find((c) => c.kind.toLowerCase() === 'video');
if (codec.mimeType.toLowerCase() === 'video/vp9') {
encodings = { scalabilityMode: 'S3T3_KEY' };
} else {
encodings = [
{ scaleResolutionDownBy: 4, maxBitrate: 500000 },
{ scaleResolutionDownBy: 2, maxBitrate: 1000000 },
{ scaleResolutionDownBy: 1, maxBitrate: 5000000 }
];
}
const videoTrack = this.localStream.getVideoTracks()[0];
this.peer.videoProducer =await this.peer.sendTransport.produce({
track: videoTrack,
encodings,
codecOptions,
codec
});
},
startRecording() {
this.Q.answer.recordingId = this.peer.externalId;
this.socket.emit("start-record", {
sessionId: this.peer.sessionId
});
},
stopRecording() {
this.socket.emit("stop-record" , {
sessionId: this.peer.sessionId
});
},
},
}
</script>
<style scoped>
.video-recorder_wrapper {
position: relative;
display: flex;
}
.video-camera {
margin: 0;
height: auto;
width: auto;
max-height: 350px;
max-width: 100%;
border-radius: 3px;
}
@media screen and (max-width: 600px) {
.video-camera {
width: calc(100% - 20px);
max-height: 600px;
}
}
</style>
server side:
There are a lot of server side code to make the signalling server. I only show the codes most related to recording.
First I create the worker and then the router and create transport using following functions.
const mediasoup = require('mediasoup');
const config = require('./config');
console.log('mediasoup loaded [version:%s]', mediasoup.version);
let workers = [];
let nextWorkerIndex = 0;
// Start the mediasoup workers
module.exports.initializeWorkers = async () => {
const { logLevel, logTags, rtcMinPort, rtcMaxPort } = config.worker;
console.log('initializeWorkers() creating %d mediasoup workers', config.numWorkers);
for (let i = 0; i < config.numWorkers; ++i) {
const worker = await mediasoup.createWorker({
logLevel, logTags, rtcMinPort, rtcMaxPort
});
worker.once('died', () => {
console.error('worker::died worker has died exiting in 2 seconds... [pid:%d]', worker.pid);
setTimeout(() => process.exit(1), 2000);
});
workers.push(worker);
}
};
module.exports.createRouter = async () => {
const worker = getNextWorker();
console.log('createRouter() creating new router [worker.pid:%d]', worker.pid);
console.log(`config.router.mediaCodecs:${JSON.stringify(config.router.mediaCodecs)}`)
return await worker.createRouter({ mediaCodecs: config.router.mediaCodecs });
};
module.exports.createTransport = async (transportType, router, options) => {
console.log('createTransport() [type:%s. options:%o]', transportType, options);
switch (transportType) {
case 'webRtc':
return await router.createWebRtcTransport(config.webRtcTransport);
case 'plain':
return await router.createPlainRtpTransport(config.plainRtpTransport);
}
};
const getNextWorker = () => {
const worker = workers[nextWorkerIndex];
if (++nextWorkerIndex === workers.length) {
nextWorkerIndex = 0;
}
return worker;
};
Then when the client side create its transports I start recording using following class. This works for Chrome but not working for Firefox.
// Class to handle child process used for running FFmpeg
const child_process = require('child_process');
const { EventEmitter } = require('events');
const fs = require('fs');
const { createSdpText } = require('./sdp');
const { convertStringToStream } = require('./utils');
const shelljs = require('shelljs');
const Recording = require('../models/Recording');
const ffmpegPath = require('@ffmpeg-installer/ffmpeg').path;
const ffmpeg = require('fluent-ffmpeg');
ffmpeg.setFfmpegPath(ffmpegPath);
module.exports = class FFmpeg {
constructor (args) {
this._rtpParameters = args;
this._process = undefined;
this._observer = new EventEmitter();
this._peer = args.peer;
this._sdpString = createSdpText(this._rtpParameters);
this._sdpStream = convertStringToStream(this._sdpString);
// create dir
const dir = process.env.REOCRDING_PATH ?? 'storage/recordings';
if (!fs.existsSync(dir)) shelljs.mkdir('-p', dir);
// create file path
this._path = `${dir}/${args.peer.sessionId}.webm`
let loop = 0;
while(fs.existsSync(this._path)) {
this._path = `${dir}/${args.peer.sessionId}-${++loop}.webm`
}
this._createProcess();
}
async _createProcess () {
// get connection model
this._recordingnModel = await Recording.findOne({sessionIds: { $in: [this._peer.sessionId] }})
this._recordingnModel.files.push(this._path);
this._recordingnModel.save();
const sdpString = this._sdpString;
const sdpStream = this._sdpStream;
// this.recordUsingFluentFFmpeg();
// return;
console.log('createProcess() [sdpString:%s]', sdpString);
this._process = child_process.spawn('ffmpeg', this._commandArgs);
this._peer.socket.emit('recording');
if (this._process.stderr) {
this._process.stderr.setEncoding('utf-8');
this._process.stderr.on('data', data =>
console.log('ffmpeg::process::data [data:%o]', data)
);
}
if (this._process.stdout) {
this._process.stdout.setEncoding('utf-8');
this._process.stdout.on('data', data =>
console.log('ffmpeg::process::data [data:%o]', data)
);
}
this._process.on('message', message =>
console.log('ffmpeg::process::message [message:%o]', message)
);
this._process.on('error', error => {
this._peer.socket.emit('recording-error');
console.error('ffmpeg::process::error [error:%o]', error)
});
this._process.once('close', () => {
this._peer.socket.emit('recording-closed');
console.log('ffmpeg::process::close');
this._observer.emit('process-close');
});
sdpStream.on('error', error =>
console.error('sdpStream::error [error:%o]', error)
);
// Pipe sdp stream to the ffmpeg process
sdpStream.resume();
sdpStream.pipe(this._process.stdin);
}
kill () {
console.log('kill() [pid:%d]', this._process.pid);
this._process.kill('SIGINT');
}
get _commandArgs () {
let commandArgs = [
'-loglevel',
'debug',
'-protocol_whitelist',
'pipe,udp,rtp',
'-fflags',
'+genpts',
'-f',
'sdp',
'-i',
'pipe:0'
];
commandArgs = commandArgs.concat(this._videoArgs);
commandArgs = commandArgs.concat(this._audioArgs);
commandArgs = commandArgs.concat([
'-f',
'webm',
'-flags',
'+global_header',
'-y',
this._path
]);
console.log('commandArgs:%o', commandArgs);
return commandArgs;
}
get _videoArgs () {
return [
'-map',
'0:v:0',
'-c:v',
'copy'
];
}
get _audioArgs () {
return [
'-map',
'0:a:0',
'-strict', // libvorbis is experimental
'-2',
'-c:a',
'copy'
];
}
recordUsingFluentFFmpeg() {
console.log(this._rtpParameters.video.rtpParameters.codec);
let proc = ffmpeg(this._sdpStream)
.inputOptions([
'-protocol_whitelist','pipe,udp,rtp',
'-f','sdp',
])
.format('webm')
.output('test.webm')
.size('720x?')
.on('start', ()=>{
console.log('Start recording')
})
.on('end', ()=>{
console.log('Stop recording')
});
proc.run();
this._process = proc;
}
}