import React, { useEffect ,useRef, useState } from 'react';
import './App.css';
import io from 'socket.io-client';
import * as mediasoupClient from 'mediasoup-client';
const socket = io('http://localhost:3001');
function VideoComponent({ stream }) {
const videoRef = useRef();
useEffect(() => {
if (videoRef.current) {
videoRef.current.srcObject = stream;
}
}, [stream]);
const Resume = ()=>{
videoRef.current.play()
}
return (
<>
<button onClick={Resume}>Resume</button>
<video ref={videoRef} autoPlay playsInline></video>
</>
);
}
function App() {
// ... rest of your socket code
const [peers, Setnewpeers] = useState([])
const videoRefs = useRef(null);
const [audioProducerCreated, setAudioProducerCreated] = useState(false);
const [videoProducerCreated, setVideoProducerCreated] = useState(false);
let roomName = 'yash'
let device
let rtpCapabilities
let producerTransport
let consumerTransports = []
let audioProducer
let videoProducer
let consumer
let isProducer = false
let params = {
// mediasoup params
encodings: [
{
rid: 'r0',
maxBitrate: 100000,
scalabilityMode: 'S1T3',
},
{
rid: 'r1',
maxBitrate: 300000,
scalabilityMode: 'S1T3',
},
{
rid: 'r2',
maxBitrate: 900000,
scalabilityMode: 'S1T3',
},
],
// https://mediasoup.org/documentation/v3/mediasoup-client/api/#ProducerCodecOptions
codecOptions: {
videoGoogleStartBitrate: 1000
}
}
let audioParams;
let videoParams = { params };
let consumingTransports = [];
socket.on('connection-success', ({ socketId }) => {
console.log(socketId)
getLocalStream()
})
const streamSuccess = (stream) => {
stream.getTracks().forEach(track => {
track.enabled = true; // Ensure track is enabled
});
audioParams = { track: stream.getAudioTracks()[0], ...audioParams };
videoParams = { track: stream.getVideoTracks()[0], ...videoParams };
joinRoom()
}
const joinRoom = () => {
socket.emit('joinRoom', { roomName }, (data) => {
console.log(`Router RTP Capabilities... ${data.rtpCapabilities}`)
// console.log(JSON.stringify(data.rtpCapabilities, null, 2))
// we assign to local variable and will be used when
// loading the client Device (see createDevice above)
rtpCapabilities = data.rtpCapabilities
// once we have rtpCapabilities from the Router, create Device
createDevice()
})
}
const getLocalStream = () => {
navigator.mediaDevices.getUserMedia({
audio: true,
video: {
width: {
min: 640,
max: 1920,
},
height: {
min: 400,
max: 1080,
}
}
})
.then(streamSuccess)
.catch(error => {
console.log(error.message)
})
}
const createDevice = async () => {
try {
device = new mediasoupClient.Device()
await device.load({
routerRtpCapabilities: rtpCapabilities
})
console.log('Device RTP Capabilities', device.rtpCapabilities)
// once the device loads, create transport
createSendTransport()
} catch (error) {
console.log(error)
if (error.name === 'UnsupportedError')
console.warn('browser not supported')
}
}
const createSendTransport = () => {
// see server's socket.on('createWebRtcTransport', sender?, ...)
// this is a call from Producer, so sender = true
socket.emit('createWebRtcTransport', { consumer: false }, ({ params }) => {
// The server sends back params needed
// to create Send Transport on the client side
if (params.error) {
console.log(params.error)
return
}
console.log(params)
producerTransport = device.createSendTransport(params)
producerTransport.on('connect', async ({ dtlsParameters }, callback, errback) => {
try {
await socket.emit('transport-connect', {
dtlsParameters,
})
// Tell the transport that parameters were transmitted.
callback()
} catch (error) {
errback(error)
}
})
producerTransport.on('produce', async (parameters, callback, errback) => {
console.log(parameters)
try {
console.log('currently producing')
if ((parameters.kind === 'audio' && audioProducerCreated) &&
(parameters.kind === 'video' && videoProducerCreated)) {
console.log(`${parameters.kind} producer already exists`);
return;
}
socket.emit('transport-produce', {
kind: parameters.kind,
rtpParameters: parameters.rtpParameters,
appData: parameters.appData,
}, ({ id, producersExist }) => {
// Tell the transport that parameters were transmitted and provide it with the
// server side producer's id.
callback({ id })
// if producers exist, then join room
if (producersExist) getProducers()
})
} catch (error) {
errback(error)
}
})
connectSendTransport()
})
}
const connectSendTransport = async () => {
// we now call produce() to instruct the producer transport
// to send media to the Router
// https://mediasoup.org/documentation/v3/mediasoup-client/api/#transport-produce
// this action will trigger the 'connect' and 'produce' events above
if (!audioProducerCreated) {
audioProducer = await producerTransport.produce(audioParams);
setAudioProducerCreated(true);
console.log('welp again audio')
// ... existing handling for audioProducer
}
if (!videoProducerCreated) {
videoProducer = await producerTransport.produce(videoParams);
console.log('welp again video')
setVideoProducerCreated(true);
// ... existing handling for videoProducer
}
audioProducer.on('trackended', () => {
console.log('audio track ended')
// close audio track
})
audioProducer.on('transportclose', () => {
console.log('audio transport ended')
// close audio track
})
videoProducer.on('trackended', () => {
console.log('video track ended')
// close video track
})
videoProducer.on('transportclose', () => {
console.log('video transport ended')
// close video track
})
}
const signalNewConsumerTransport = async (remoteProducerId) => {
//check if we are already consuming the remoteProducerId
if (consumingTransports.includes(remoteProducerId)) return;
consumingTransports.push(remoteProducerId);
console.log('signaling new users')
await socket.emit('createWebRtcTransport', { consumer: true }, ({ params }) => {
// The server sends back params needed
// to create Send Transport on the client side
if (params.error) {
console.log(params.error)
return
}
console.log(`PARAMS... ${params}`)
let consumerTransport
try {
consumerTransport = device.createRecvTransport(params)
} catch (error) {
// exceptions:
// {InvalidStateError} if not loaded
// {TypeError} if wrong arguments.
console.log(error)
return
}
consumerTransport.on('connect', async ({ dtlsParameters }, callback, errback) => {
try {
// Signal local DTLS parameters to the server side transport
// see server's socket.on('transport-recv-connect', ...)
await socket.emit('transport-recv-connect', {
dtlsParameters,
serverConsumerTransportId: params.id,
})
// Tell the transport that parameters were transmitted.
callback()
} catch (error) {
// Tell the transport that something was wrong
errback(error)
}
})
connectRecvTransport(consumerTransport, remoteProducerId, params.id)
})
}
const getProducers = () => {
socket.emit('getProducers', producerIds => {
console.log(producerIds)
// for each of the producer create a consumer
// producerIds.forEach(id => signalNewConsumerTransport(id))
producerIds.forEach(signalNewConsumerTransport)
})
}
const connectRecvTransport = async (consumerTransport, remoteProducerId, serverConsumerTransportId) => {
// for consumer, we need to tell the server first
// to create a consumer based on the rtpCapabilities and consume
// if the router can consume, it will send back a set of params as below
console.log('Consuming rn')
socket.emit('consume', {
rtpCapabilities: device.rtpCapabilities,
remoteProducerId,
serverConsumerTransportId,
}, async ({ params }) => {
if (params.error) {
console.log('Cannot Consume')
return
}
console.log(`Consumer Params ${params}`)
// then consume with the local consumer transport
// which creates a consumer
const consumer = await consumerTransport.consume({
id: params.id,
producerId: params.producerId,
kind: params.kind,
rtpParameters: params.rtpParameters
})
consumerTransports = [
...consumerTransports,
{
consumerTransport,
serverConsumerTransportId: params.id,
producerId: remoteProducerId,
consumer,
},
]
const { track } = consumer
console.log(track)
const newStream = new MediaStream([track]);
Setnewpeers(prevPeers => [...prevPeers, newStream]);
// the server consumer started with media paused
// so we need to inform the server to resume
socket.emit('consumer-resume', { serverConsumerId: params.serverConsumerId })
})
}
socket.on('producer-closed', ({ remoteProducerId }) => {
// server notification is received when a producer is closed
// we need to close the client-side consumer and associated transport
const producerToClose = consumerTransports.find(transportData => transportData.producerId === remoteProducerId)
producerToClose.consumerTransport.close()
producerToClose.consumer.close()
// remove the consumer transport from the list
consumerTransports = consumerTransports.filter(transportData => transportData.producerId !== remoteProducerId)
// remove the video div element
})
useEffect(() => {
console.log(peers)
}, [peers])
return (
<div id="video">
{peers.map((stream, index) => (
<VideoComponent key={index} stream={stream} />
))}
</div>
);
}
export default App;
const mediasoup = require('mediasoup');
const fs = require('fs');
const path = require('path');
module.exports = async function(io) {
let worker
let rooms = {} // { roomName1: { Router, rooms: [ sicketId1, ... ] }, ...}
let peers = {} // { socketId1: { roomName1, socket, transports = [id1, id2,] }, producers = [id1, id2,] }, consumers = [id1, id2,], peerDetails }, ...}
let transports = [] // [ { socketId1, roomName1, transport, consumer }, ... ]
let producers = [] // [ { socketId1, roomName1, producer, }, ... ]
let consumers = [] // [ { socketId1, roomName1, consumer, }, ... ]
const mediaCodecs = [
{
kind: 'audio',
mimeType: 'audio/opus',
clockRate: 48000,
channels: 2,
},
{
kind: 'video',
mimeType: 'video/VP8',
clockRate: 90000,
parameters: {
'x-google-start-bitrate': 1000,
},
},
];
const createWorker = async () => {
const worker = await mediasoup.createWorker({
rtcMinPort: 2000,
rtcMaxPort: 2100,
});
console.log(`worker pid ${worker.pid}`);
worker.on('died', error => {
// This implies something serious happened, so kill the application
console.error('mediasoup worker has died');
setTimeout(() => process.exit(1), 2000); // exit in 2 seconds
});
return worker;
};
worker = await createWorker(); // Wait for the worker creation to complete
router = await worker.createRouter({ mediaCodecs });
const createWebRtcTransport = async (callback) => {
try {
// https://mediasoup.org/documentation/v3/mediasoup/api/#WebRtcTransportOptions
const webRtcTransport_options = {
listenIps: [
{
ip: '127.0.0.1', // replace with relevant IP address
}
],
enableUdp: true,
enableTcp: true,
preferUdp: true,
}
// https://mediasoup.org/documentation/v3/mediasoup/api/#router-createWebRtcTransport
let transport = await router.createWebRtcTransport(webRtcTransport_options)
console.log(`transport id: ${transport.id}`)
transport.on('dtlsstatechange', dtlsState => {
if (dtlsState === 'closed') {
transport.close()
}
})
transport.on('close', () => {
console.log('transport closed')
})
// send back to the client the following prameters
callback({
// https://mediasoup.org/documentation/v3/mediasoup-client/api/#TransportOptions
params: {
id: transport.id,
iceParameters: transport.iceParameters,
iceCandidates: transport.iceCandidates,
dtlsParameters: transport.dtlsParameters,
}
})
return transport
} catch (error) {
console.log(error)
callback({
params: {
error: error
}
})
}
}
const informConsumers = (roomName, socketId, id) => {
console.log(`just joined, id ${id} ${roomName}, ${socketId}`)
// A new producer just joined
// let all consumers to consume this producer
producers.forEach(producerData => {
if (producerData.socketId !== socketId && producerData.roomName === roomName) {
const producerSocket = peers[producerData.socketId].socket
// use socket to send producer id to producer
producerSocket.emit('new-producer', { producerId: id })
}
})
}
const getTransport = (socketId) => {
const transportData = transports.find(transport =>
transport.socketId === socketId && !transport.consumer);
if (!transportData) {
console.error(`Transport not found for socket ID: ${socketId}`);
return null;
}
return transportData.transport;
};
function cleanUpPeer(socketId) {
// Clean up transports, producers, and consumers associated with the socketId
transports = removeItems(transports, socketId, 'transport');
producers = removeItems(producers, socketId, 'producer');
consumers = removeItems(consumers, socketId, 'consumer');
// Remove the peer from the peers list
const roomName = peers[socketId]?.roomName;
delete peers[socketId];
// Update the room's peers list
if (roomName && rooms[roomName]) {
rooms[roomName].peers = rooms[roomName].peers.filter(id => id !== socketId);
}
}
const removeItems = (items, socketId, type) => {
items.forEach(item => {
if (item.socketId === socketId && item[type] && typeof item[type].close === 'function') {
item[type].close();
}
});
items = items.filter(item => item.socketId !== socketId);
return items;
};
io.on('connection', (socket) => {
try{
if (peers[socket.id]) {
console.log(`User already connected: ${socket.id}`);
return;
}
console.log('Socketid is ', socket.id)
console.log('A user connected');
socket.emit('connection-success', { socketID: socket.id });
socket.on('disconnect', () => {
// do some cleanup
console.log('peer disconnected')
cleanUpPeer(socket.id);
})
socket.on('joinRoom', async ({ roomName }, callback) => {
const router1 = await createRoom(roomName, socket.id)
peers[socket.id] = {
socket,
roomName,
transports: [],
producers: [],
consumers: [],
peerDetails: {
name: '',
isAdmin: false,
}
}
// get Router RTP Capabilities
const rtpCapabilities = router1.rtpCapabilities
// call callback from the client and send back the rtpCapabilities
callback({ rtpCapabilities })
})
const createRoom = async (roomName, socketId) => {
// worker.createRouter(options)
// options = { mediaCodecs, appData }
// mediaCodecs -> defined above
// appData -> custom application data - we are not supplying any
// none of the two are required
let router1
let peers = []
if (rooms[roomName]) {
router1 = rooms[roomName].router
peers = rooms[roomName].peers || []
} else {
router1 = await worker.createRouter({ mediaCodecs, })
}
console.log(`Router ID: ${router1.id}`, peers.length)
rooms[roomName] = {
router: router1,
peers: [...peers, socketId],
}
return router1
}
socket.on('createWebRtcTransport', async ({ consumer }, callback) => {
// get Room Name from Peer's properties
const roomName = peers[socket.id].roomName
// get Router (Room) object this peer is in based on RoomName
const router = rooms[roomName].router
createWebRtcTransport(router).then(
transport => {
callback({
params: {
id: transport.id,
iceParameters: transport.iceParameters,
iceCandidates: transport.iceCandidates,
dtlsParameters: transport.dtlsParameters,
}
})
// add transport to Peer's properties
addTransport(transport, roomName, consumer)
},
error => {
console.log(error)
})
})
const addTransport = (transport, roomName, consumer) => {
transports = [
...transports,
{ socketId: socket.id, transport, roomName, consumer, }
]
peers[socket.id] = {
...peers[socket.id],
transports: [
...peers[socket.id].transports,
transport.id,
]
}
}
const addProducer = (producer, roomName) => {
producers = [
...producers,
{ socketId: socket.id, producer, roomName, }
]
peers[socket.id] = {
...peers[socket.id],
producers: [
...peers[socket.id].producers,
producer.id,
]
}
}
const addConsumer = (consumer, roomName) => {
// add the consumer to the consumers list
consumers = [
...consumers,
{ socketId: socket.id, consumer, roomName, }
]
// add the consumer id to the peers list
peers[socket.id] = {
...peers[socket.id],
consumers: [
...peers[socket.id].consumers,
consumer.id,
]
}
}
socket.on('getProducers', callback => {
//return all producer transports
const { roomName } = peers[socket.id]
console.log(producers)
let producerList = []
producers.forEach(producerData => {
if (producerData.socketId == socket.id && producerData.roomName === roomName) {
producerList = [...producerList, producerData.producer.id];
}
})
// return the producer list back to the client
callback(producerList)
})
socket.on('transport-produce', async ({ kind, rtpParameters, appData }, callback) => {
// call produce based on the prameters from the client
try{
let producer
const { roomName } = peers[socket.id];
const existingProducer = producers.find(
producerData => producerData.socketId === socket.id &&
producerData.roomName === roomName &&
producerData.producer.kind === kind
);
// Check if the peer already has a producer of the same kind
if (existingProducer) {
console.log(`Peer already has a ${kind} producer with ID ${existingProducer.producer.id}`);
callback({ id: existingProducer.producer.id, producersExist: producers.length > 1 ? true : false });
return;
}
if(!existingProducer){
producer = await getTransport(socket.id).produce({
kind,
rtpParameters,
})
}
// Retrieve transport and validate its existence
const transport = getTransport(socket.id);
if (!transport) {
console.error('Transport not found.');
// Handle transport not found error
return;
}
// add producer to the producers array
addProducer(producer, roomName)
informConsumers(roomName, socket.id, producer.id)
console.log('Producer ID: ', producer.id, producer.kind)
producer.on('transportclose', () => {
console.log('transport for this producer closed ')
producer.close()
})
// Send back to the client the Producer's id
callback({
id: producer.id,
producersExist: producers.length>1 ? true : false
})
}catch(error){
console.log(error)
}
})
socket.on('transport-recv-connect', async ({ dtlsParameters, serverConsumerTransportId }) => {
console.log(`DTLS PARAMS: ${dtlsParameters}`)
const consumerTransport = transports.find(transportData => (
transportData.consumer && transportData.transport.id == serverConsumerTransportId
)).transport
await consumerTransport.connect({ dtlsParameters })
})
socket.on('consume', async ({ rtpCapabilities, remoteProducerId, serverConsumerTransportId }, callback) => {
try {
const { roomName } = peers[socket.id]
const router = rooms[roomName].router
let consumerTransport = transports.find(transportData => (
transportData.consumer && transportData.transport.id == serverConsumerTransportId
)).transport
// check if the router can consume the specified producer
if (router.canConsume({
producerId: remoteProducerId,
rtpCapabilities,
})) {
// transport can now consume and return a consumer
const consumer = await consumerTransport.consume({
producerId: remoteProducerId,
rtpCapabilities,
})
console.log(consumer.kind)
console.log(consumer)
consumer.on('transportclose', () => {
console.log('transport close from consumer')
})
consumer.on('producerclose', () => {
console.log('producer of consumer closed')
socket.emit('producer-closed', { remoteProducerId })
consumerTransport.close([])
transports = transports.filter(transportData => transportData.transport.id !== consumerTransport.id)
consumer.close()
consumers = consumers.filter(consumerData => consumerData.consumer.id !== consumer.id)
})
socket.on('consumer-resume', async ({ serverConsumerId }) => {
console.log('consumer resume');
const consumerData = consumers.find(consumerData => consumerData.consumer.id === serverConsumerId);
if (consumerData) {
const { consumer } = consumerData;
await consumer.resume();
} else {
console.error(`Consumer with ID ${serverConsumerId} not found.`);
}
});
addConsumer( consumer, roomName)
// from the consumer extract the following params
// to send back to the Client
const params = {
id: consumer.id,
producerId: remoteProducerId,
kind: consumer.kind,
rtpParameters: consumer.rtpParameters,
serverConsumerId: consumer.id,
}
// send the parameters to the client
callback({ params })
}
} catch (error) {
console.log(error.message)
callback({
params: {
error: error
}
})
}
})
const createWebRtcTransport = async (router) => {
return new Promise(async (resolve, reject) => {
try {
// https://mediasoup.org/documentation/v3/mediasoup/api/#WebRtcTransportOptions
const webRtcTransport_options = {
listenIps: [
{
ip: '127.0.0.1', // replace with relevant IP address
}
],
enableUdp: true,
enableTcp: true,
preferUdp: true,
}
// https://mediasoup.org/documentation/v3/mediasoup/api/#router-createWebRtcTransport
let transport = await router.createWebRtcTransport(webRtcTransport_options)
console.log(`transport id: ${transport.id}`)
transport.on('dtlsstatechange', dtlsState => {
if (dtlsState === 'closed') {
transport.close()
}
})
transport.on('close', () => {
console.log('transport closed')
})
resolve(transport)
} catch (error) {
reject(error)
}
})
}
}catch (error) {
console.error('Error:', error);
}
});
};
i am not sure, i tried gpt 4 with plugins, tried reading through docs aswell, but i cant seem to i find what is wrong
There can be multiple reasons for this:
- If you are running it on server and haven’t set the proper public ip address of server in mediasoup config
- If you haven’t open tcp, udp ports to public which you provided to mediasoup
- The consumer may be paused, you may have forgot to resume it
- The producer is not produced properly.
To get to which of the above reasons is in your case you will have to debug it, one of the easiest ways to do that is use chrome browser, initiate the call and go to chrome://webrtc-internals, you will see many details there leading you to the source of the issue
- You can see the connection state to see if it is connected
- You can see the candidate grid to see if there is issue with ice candidates, you can look for the server public ip, port from there and see if that is causing the issue
- You can see the outgoing, incoming bytes in producer, consumer side which can show you if either producer is produced properly or is it the issue on your consumer side.
Thanks for taking the time out for helping me means alot to me,
1.yes the server ip is set properly, shows same in internals too
2.yes i have opened the specified ports for udp and tcp
3.my server keeps trying to resume the track multiple times, so i think it might not be the issue here,
4. producer seems to be working properly, it produces a media stream of video and audio, and i also see bytes being sent in webrtc internals
5. yes the connection state, ice says connected
6. i did check the grid,
7. the outgoing bits seem fine, and packets aswell, the receiving bits seems to be the issue which might say my consumer is the issue here? I tried to debug the issue by adding alot of logs, but everything seems to be working fine, maybe its the way i am assigning the media to the element? although it seems to be correct, i tried using state approach but got same result
if you find something that might be causing that do point it out I would appreciate it, i will keep trying Thanks again
- producer seems to be fine
- can you show the stats report for consumer transport as well from webrtc-internals?
- make sure you are not consuming and producing on the same transport, a transport can either be producing or consuming. You should have producers, consumers on different transports.
- autoplay doesn’t seems to be the issue, if track’s muted prop is true then it means data is not being received. make sure the consumer is resumed properly.
i have a logic to create separate transports for producer and consumer so they should not be on same transport,
I am still little confued on the resumed property to be honest, i am getting log messages that its been triggered Multiple times
These stat graphs are of outbound-rtp which is for producer, kindly show the stats for inbound rtp of for consumer transport overall.
You need to check with the outgoing bytes from mediasoup side as well, mediasoup provided all the thing to check how much data is being delivered to consumer and whether consumer is in resumed state or not. Just put some console logs on relevant events and it will give you the full picture.
These are the mediasoup logs, they say dtls not connnected, and prints them every 1-2 secs, tried to do some searching around and found similar issue although the person did not post how they fixed it (they said they passed dtls params wrong, but i checked in docs and i am passing the params correctly,
I dont seem to have a inbound transport, unless i am looking at something wrong here,
That error “MID already exists” seems to be the cause, this can happen if you are creating same producer twice on the same transport like with same parameters. You can search of this error in this forum, it will give you the insights. Fix the issue that is causing the error then it should clear the blockage.
Thankyou so much for you answer, yea this fixed the issue, had a different issue after that, managed to get it working, thank you, you helped me save so much time
1 Like
Great, glad to hear that 