Video stream from mediasoup server to react-native-webrtc client

Integration
1

Hi!

I am trying to create an application that has a mediasoup server publishing a video stream from local images injected via ffmpeg and a react-native-client consuming the video stream. I did my best accommodating all functions such that it should comply with mediasoup documentation, however, I am experiencing a problem with ICE negotiation.

Here is the server code:

const WebSocket = require('ws');
const fs = require('fs');
const ffmpeg = require('fluent-ffmpeg');
const { PassThrough } = require('stream');
const mediasoup = require('mediasoup');

const server = new WebSocket.Server({ port: 8080 });

const mediasoupOptions = {
  worker: {
    rtcMinPort: 10000,
    rtcMaxPort: 10100,
  },
  router: {
    mediaCodecs: [
      {
        kind: 'video',
        mimeType: 'video/h264',
        clockRate: 90000,
        parameters: {
          'packetization-mode': 1,
          'profile-level-id': '42e01f',
          'level-asymmetry-allowed': 1,
        },
      },
    ],
  },
};

(async () => {
  const worker = await mediasoup.createWorker(mediasoupOptions.worker);

  worker.on('died', () => {
    console.error('mediasoup Worker died, exiting...');
    process.exit(1);
  });

  const router = await worker.createRouter(mediasoupOptions.router);
  let currentChunk = null;

  const inputStream = new PassThrough();
  const outputStream = new PassThrough();

  const videoStream = ffmpeg()
    .addInput(inputStream)
    .inputFPS(30)
    .inputFormat('image2pipe')
    .inputOptions(['-pix_fmt yuv420p'])
    .videoCodec('libx264')
    .size('640x480')
    .format('rtp')
    .output(outputStream)
    .on('error', (err) => console.error('ffmpeg error:', err));

  videoStream.run();

  outputStream.on('data', (chunk) => {
    currentChunk = chunk;
  });

  server.on('connection', async (socket) => {
    console.log('Client connected');

    const transport = await router.createWebRtcTransport({
      listenIps: [{ ip: '192.168.178.108' }],
      enableSctp: true,
      enableUdp: true,
      enableTcp: true,
      preferUdp: true,
      initialAvailableOutgoingBitrate: 1000000,
    });

    socket.on('message', async (message) => {
        console.log('Received message:', message); 
      const { action, data } = JSON.parse(message);

      console.log("action: ", action)

      console.log("data", data)

      if (action === 'transport-connect') {
        // Debugging: Check if the transport is created
        console.log('Transport created:', transport);

        // Listen for the 'icestatechange' event and log the changes
        transport.on('icestatechange', (iceState) => {
          console.log(`transport iceState changed to: ${iceState}`);
        });

        transport.on('routerclose', () => {
          console.log('transport "routerclose" event triggered');
          socket.send(JSON.stringify({ action: 'transport-router-close' }));
        });
        

        transport.on('dtlsstatechange', async (dtlsState) => {
            console.log(`transport dtlsState changed to: ${dtlsState}`);
            if (dtlsState === 'connected') {
              console.log('DTLS connected, calling transport.connect');
              await transport.connect({ dtlsParameters: data.dtlsParameters });
              const codec = { ...mediasoupOptions.router.mediaCodecs[0], payloadType: 102 };
              const producer = await transport.produce({ kind: 'video', rtpParameters: { codecs: [codec], encodings: [] } });
          
              // Send the correct rtpParameters to the client
              socket.send(JSON.stringify({
                action: 'transport-connected',
                data: {
                  id: producer.id,
                  rtpParameters: producer.rtpParameters
                }
              }));
            }
          });

        
        // Debugging: Check if the transport is connecting
        console.log('Transport connecting:', transport);
      } else if (action === 'produce') {
        console.log('Received produce action'); 
        if (currentChunk) {
            console.log('Producidtlsstatechangeng with currentChunk');
          // Create an RtpStream for the video stream
          const rtpStream = {
            ssrc: Math.floor(Math.random() * 0xffffffff),
            payloadType: 102, // Use a dynamic payload type
            mimeType: 'video/h264',
            clockRate: 90000,
            payload: currentChunk,
          };

          const codec = { ...mediasoupOptions.router.mediaCodecs[0], payloadType: rtpStream.payloadType };

          const producer = await transport.produce({ kind: 'video', rtpParameters: { codecs: [codec], encodings: [rtpStream] } });
          socket.send(JSON.stringify({ action: 'produced', data: { id: producer.id } }));
        }
      } else if (action === 'transport-ready') {
        sendNextImage();
      }
    });

    socket.send(
        JSON.stringify({
          action: 'transport-create',
          data: {
            id: transport.id,
            iceParameters: transport.iceParameters,
            iceCandidates: transport.iceCandidates,
            dtlsParameters: transport.dtlsParameters,
            sctpParameters: transport.sctpParameters,
            routerRtpCapabilities: router.rtpCapabilities,
          },
        })
      );

    const images = [];
    for (let i = 1; i <= 60; i++) {
      const imageIndex = String(i).padStart(5, '0');
      images.push(`data/video-frames/frame_${imageIndex}.jpg`);
    }

    async function sendNextImage() {
      if (transport.dtlsState === 'connected') {
        console.log('Sending images');
        for (const image of images) {
          const imageData = await fs.promises.readFile(image);
          inputStream.write(imageData);
          await new Promise((resolve) => setTimeout(resolve, 1000 / 30));
        }
      }
      if (socket.readyState === WebSocket.OPEN) {
        setTimeout(sendNextImage, 1000 / 30);
      }
    }

    socket.on('close', () => {
      console.log('Client disconnected');
      transport.close();
    });
  });
})();

console.log('Server listening on port 8080');

And here is the client code:

import React, { useEffect, useState } from 'react';
import { StyleSheet, View } from 'react-native';
import * as mediasoupClient from 'mediasoup-client';
import { registerGlobals } from 'react-native-webrtc';
import { RTCView } from 'react-native-webrtc';

registerGlobals(); // Register WebRTC globals

const App = () => {
  const [socket, setSocket] = useState(null);
  const [videoTrack, setVideoTrack] = useState(null);

  WebSocket.prototype.sendAsync = function (data) {
    return new Promise((resolve, reject) => {
      this.send(JSON.stringify(data), (err) => {
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      });
    });
  };

  let device;

  useEffect(() => {
    const setupWebrtc = async (data) => {
      let recvTransport;

      device = new mediasoupClient.Device();

      await device.load({ routerRtpCapabilities: data.routerRtpCapabilities });
      

      recvTransport = device.createRecvTransport({
        id: data.id,
        iceParameters: data.iceParameters,
        iceCandidates: data.iceCandidates,
        dtlsParameters: data.dtlsParameters,
        sctpParameters: data.sctpParameters,
      });
      
      recvTransport.on('connectionstatechange', async (state) => {
        console.log(`recvTransport connection state changed to: ${state}`);
        if (state === 'connected') {
          ws.sendAsync({ action: 'transport-ready' });
        }
      });

      recvTransport.on('dtlsstatechange', (state) => {
        console.log(`recvTransport DTLS state changed to: ${state}`);
      });

      recvTransport.on('icestatechange', (state) => {
        console.log(`recvTransport ICE state changed to: ${state}`);
      });

      // Send the transport-connect message
      console.log("dtlsParameters:", recvTransport._handler._remoteSdp._dtlsParameters);
      await ws.sendAsync({
        action: 'transport-connect',
        data: { dtlsParameters: recvTransport._handler._remoteSdp._dtlsParameters },
      });
    };

    const ws = new WebSocket('ws://192.168.178.108:8080');

    ws.onopen = async () => {
      console.log('Connected to server');
      setSocket(ws);
    };

    ws.onmessage = async (event) => {
      const { action, data } = JSON.parse(event.data);

      if (action === 'transport-create') {
        console.log("ws.onmessage action === 'transport-create'");
        console.log("data: ", data)
        setupWebrtc(data);
        
      } else if (action === 'transport-connected') {
        const { id, rtpParameters } = data;

        console.log("ws.onmessage action === 'transport-connected'");

        const stream = await recvTransport.consume({
          producerId: id, // Pass the received producerId to the consume method
          rtpCapabilities: device.rtpCapabilities, // RTP capabilities of the consuming endpoint
          rtpParameters: rtpParameters,
          paused: true,
          enableRtx: true, // Set enableRtx option as specified in the documentation
        });
        console.log('Received stream:', stream);

        // Set the video track in the state
        setVideoTrack(stream.track);

        // Send the 'produce' action to the server
        await ws.sendAsync({
          action: 'produce',
          data: { id: stream.id, kind: 'video', rtpParameters: stream.rtpParameters },
        });
      } else if (action === 'transport-router-close') {
        console.log("ws.onmessage action === 'transport-router-close'");
        recvTransport.close();
      }
    };

    ws.onerror = (event) => {
      console.error('WebSocket error:', event);
    };

    ws.onclose = (event) => {
      console.log('Disconnected from server', event);
      // Clean up transports and close the device
      device.close();
    };
  }, []);

  return (
    <View style={styles.container}>
      {videoTrack && (
        <RTCView
          style={styles.video}
          streamURL={videoTrack.toURL()}
          objectFit="cover"
        />
      )}
    </View>
  );
};

const styles = StyleSheet.create({
  container: {
    flex: 1,
    justifyContent: 'center',
    alignItems: 'center',
    backgroundColor: '#F5FCFF',
  },
  video: {
    width: '100%',
    height: '100%',
  },
});

export default App;

The client manages to connect to the server. Also, the “transport-connect” and “ransport-create” are sent. However, the “on(‘icestatechange’”) event fails to trigger on either the server or client.

Let me provide also the full logs.

Server logs:

Server listening on port 8080
Client connected
Received message: <Buffer 7b 22 61 63 74 69 6f 6e 22 3a 22 74 72 61 6e 73 70 6f 72 74 2d 63 6f 6e 6e 65 63 74 22 2c 22 64 61 74 61 22 3a 7b 22 64 74 6c 73 50 61 72 61 6d 65 74 ... 783 more bytes>
action:  transport-connect
data {
  dtlsParameters: {
    fingerprints: [ [Object], [Object], [Object], [Object], [Object] ],
    role: 'auto'
  }
}
Transport created: WebRtcTransport {
  _events: [Object: null prototype] {
    '@close': [Function (anonymous)],
    '@listenserverclose': [Function (anonymous)],
    '@newproducer': [Function (anonymous)],
    '@producerclose': [Function (anonymous)],
    '@newdataproducer': [Function (anonymous)],
    '@dataproducerclose': [Function (anonymous)]
  },
  _eventsCount: 6,
  _maxListeners: Infinity,
  internal: {
    routerId: '8b01e899-3b3a-48bb-b389-5bb5fd702e70',
    transportId: '321461af-b1f7-47db-ac19-acb6324d5a9f'
  },
  channel: Channel {
    _events: [Object: null prototype] {
      '321461af-b1f7-47db-ac19-acb6324d5a9f': [Function (anonymous)]
    },
    _eventsCount: 1,
    _maxListeners: Infinity,
    [Symbol(kCapture)]: false
  },
  payloadChannel: PayloadChannel {
    _events: [Object: null prototype] {},
    _eventsCount: 0,
    _maxListeners: Infinity,
    [Symbol(kCapture)]: false
  },
  getProducerById: [Function: getProducerById],
  getDataProducerById: [Function: getDataProducerById],
  consumers: Map(0) {},
  dataProducers: Map(0) {},
  dataConsumers: Map(0) {},
  [Symbol(kCapture)]: false
}
Transport connecting: WebRtcTransport {
  _events: [Object: null prototype] {
    '@close': [Function (anonymous)],
    '@listenserverclose': [Function (anonymous)],
    '@newproducer': [Function (anonymous)],
    '@producerclose': [Function (anonymous)],
    '@newdataproducer': [Function (anonymous)],
    '@dataproducerclose': [Function (anonymous)],
    icestatechange: [Function (anonymous)],
    routerclose: [Function (anonymous)],
    dtlsstatechange: [AsyncFunction (anonymous)]
  },
  _eventsCount: 9,
  _maxListeners: Infinity,
  internal: {
    routerId: '8b01e899-3b3a-48bb-b389-5bb5fd702e70',
    transportId: '321461af-b1f7-47db-ac19-acb6324d5a9f'
  },
  channel: Channel {
    _events: [Object: null prototype] {
      '321461af-b1f7-47db-ac19-acb6324d5a9f': [Function (anonymous)]
    },
    _eventsCount: 1,
    _maxListeners: Infinity,
    [Symbol(kCapture)]: false
  },
  payloadChannel: PayloadChannel {
    _events: [Object: null prototype] {},
    _eventsCount: 0,
    _maxListeners: Infinity,
    [Symbol(kCapture)]: false
  },
  getProducerById: [Function: getProducerById],
  getDataProducerById: [Function: getDataProducerById],
  consumers: Map(0) {},
  dataProducers: Map(0) {},
  dataConsumers: Map(0) {},
  [Symbol(kCapture)]: false
}

Client logs:


 LOG  Running "WebrtcImageApp" with {"rootTag":11}
 LOG  Connected to server
 LOG  ws.onmessage action === 'transport-create'
 INFO  mediasoup-client:Device constructor() +0ms
 INFO  mediasoup-client:Device this._detectDevice() | ReactNative UnifiedPlan handler chosen +0ms
 INFO  mediasoup-client:Device constructor() | detected handler: ReactNativeUnifiedPlan +1ms
 INFO  mediasoup-client:ReactNativeUnifiedPlan close() +0ms
 INFO  mediasoup-client:Device load() [routerRtpCapabilities:'{"codecs": [{"clockRate": 90000, "kind": "video", "mimeType": "video/H264", "parameters": [Object], "preferredPayloadType": 100, "rtcpFeedback": [Array]}, {"clockRate": 90000, "kind": "video", "mimeType": "video/rtx", "parameters": [Object], "preferredPayloadType": 101, "rtcpFeedback": [Array]}], "headerExtensions": [{"direction": "sendrecv", "kind": "audio", "preferredEncrypt": false, "preferredId": 1, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 1, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"direction": "recvonly", "kind": "video", "preferredEncrypt": false, "preferredId": 2, "uri": "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"}, {"direction": "recvonly", "kind": "video", "preferredEncrypt": false, "preferredId": 3, "uri": "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"}, {"direction": "sendrecv", "kind": "audio", "preferredEncrypt": false, "preferredId": 4, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 4, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"direction": "recvonly", "kind": "audio", "preferredEncrypt": false, "preferredId": 5, "uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 5, "uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 6, "uri": "http://tools.ietf.org/html/draft-ietf-avtext-framemarking-07"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 7, "uri": "urn:ietf:params:rtp-hdrext:framemarking"}, {"direction": "sendrecv", "kind": "audio", "preferredEncrypt": false, "preferredId": 10, "uri": "urn:ietf:params:rtp-hdrext:ssrc-audio-level"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 11, "uri": "urn:3gpp:video-orientation"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 12, "uri": "urn:ietf:params:rtp-hdrext:toffset"}, {"direction": "sendrecv", "kind": "audio", "preferredEncrypt": false, "preferredId": 13, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 13, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time"}]}'] +7ms
 INFO  mediasoup-client:ReactNativeUnifiedPlan getNativeRtpCapabilities() +12ms
 LOG  rn-webrtc:pc:DEBUG 0 ctor +0ms
 LOG  rn-webrtc:pc:DEBUG 0 addTransceiver +1ms
 LOG  rn-webrtc:pc:DEBUG 0 addTransceiver +8ms
 LOG  rn-webrtc:pc:DEBUG 0 createOffer +4ms
 LOG  rn-webrtc:pc:DEBUG 0 createOffer OK +5ms
 LOG  rn-webrtc:pc:DEBUG 0 close +0ms
 INFO  mediasoup-client:Device load() | got native RTP capabilities:'{"codecs": [{"channels": 2, "clockRate": 48000, "kind": "audio", "mimeType": "audio/opus", "parameters": [Object], "preferredPayloadType": 111, "rtcpFeedback": [Array]}, {"channels": 2, "clockRate": 48000, "kind": "audio", "mimeType": "audio/red", "parameters": [Object], "preferredPayloadType": 63, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 8000, "kind": "audio", "mimeType": "audio/G722", "parameters": [Object], "preferredPayloadType": 9, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 8000, "kind": "audio", "mimeType": "audio/ILBC", "parameters": [Object], "preferredPayloadType": 102, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 8000, "kind": "audio", "mimeType": "audio/PCMU", "parameters": [Object], "preferredPayloadType": 0, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 8000, "kind": "audio", "mimeType": "audio/PCMA", "parameters": [Object], "preferredPayloadType": 8, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 8000, "kind": "audio", "mimeType": "audio/CN", "parameters": [Object], "preferredPayloadType": 13, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 48000, "kind": "audio", "mimeType": "audio/telephone-event", "parameters": [Object], "preferredPayloadType": 110, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 8000, "kind": "audio", "mimeType": "audio/telephone-event", "parameters": [Object], "preferredPayloadType": 126, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/VP8", "parameters": [Object], "preferredPayloadType": 98, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/rtx", "parameters": [Object], "preferredPayloadType": 99, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/AV1", "parameters": [Object], "preferredPayloadType": 39, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/rtx", "parameters": [Object], "preferredPayloadType": 40, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/VP9", "parameters": [Object], "preferredPayloadType": 100, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/rtx", "parameters": [Object], "preferredPayloadType": 101, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/VP9", "parameters": [Object], "preferredPayloadType": 127, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/rtx", "parameters": [Object], "preferredPayloadType": 103, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/red", "parameters": [Object], "preferredPayloadType": 104, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/rtx", "parameters": [Object], "preferredPayloadType": 105, "rtcpFeedback": [Array]}, {"channels": undefined, "clockRate": 90000, "kind": "video", "mimeType": "video/ulpfec", "parameters": [Object], "preferredPayloadType": 106, "rtcpFeedback": [Array]}], "headerExtensions": [{"kind": "audio", "preferredId": 1, "uri": "urn:ietf:params:rtp-hdrext:ssrc-audio-level"}, {"kind": "audio", "preferredId": 2, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"kind": "audio", "preferredId": 3, "uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"}, {"kind": "audio", "preferredId": 4, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"kind": "video", "preferredId": 14, "uri": "urn:ietf:params:rtp-hdrext:toffset"}, {"kind": "video", "preferredId": 2, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"kind": "video", "preferredId": 13, "uri": "urn:3gpp:video-orientation"}, {"kind": "video", "preferredId": 3, "uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"}, {"kind": "video", "preferredId": 5, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"}, {"kind": "video", "preferredId": 6, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/video-content-type"}, {"kind": "video", "preferredId": 7, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/video-timing"}, {"kind": "video", "preferredId": 8, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/color-space"}, {"kind": "video", "preferredId": 4, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"kind": "video", "preferredId": 10, "uri": "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"}, {"kind": "video", "preferredId": 11, "uri": "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"}]}' +43ms
 INFO  mediasoup-client:Device load() | got extended RTP capabilities:'{"codecs": [], "headerExtensions": [{"direction": "sendrecv", "encrypt": false, "kind": "audio", "recvId": 1, "sendId": 4, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"direction": "sendrecv", "encrypt": false, "kind": "video", "recvId": 1, "sendId": 4, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"direction": "sendonly", "encrypt": false, "kind": "video", "recvId": 2, "sendId": 10, "uri": "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"}, {"direction": "sendonly", "encrypt": false, "kind": "video", "recvId": 3, "sendId": 11, "uri": "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id"}, {"direction": "sendrecv", "encrypt": false, "kind": "audio", "recvId": 4, "sendId": 2, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"direction": "sendrecv", "encrypt": false, "kind": "video", "recvId": 4, "sendId": 2, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"direction": "sendonly", "encrypt": false, "kind": "audio", "recvId": 5, "sendId": 3, "uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"}, {"direction": "sendrecv", "encrypt": false, "kind": "video", "recvId": 5, "sendId": 3, "uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"}, {"direction": "sendrecv", "encrypt": false, "kind": "audio", "recvId": 10, "sendId": 1, "uri": "urn:ietf:params:rtp-hdrext:ssrc-audio-level"}, {"direction": "sendrecv", "encrypt": false, "kind": "video", "recvId": 11, "sendId": 13, "uri": "urn:3gpp:video-orientation"}, {"direction": "sendrecv", "encrypt": false, "kind": "video", "recvId": 12, "sendId": 14, "uri": "urn:ietf:params:rtp-hdrext:toffset"}]}' +5ms
 INFO  mediasoup-client:Device load() | got receiving RTP capabilities:'{"codecs": [], "headerExtensions": [{"direction": "sendrecv", "kind": "audio", "preferredEncrypt": false, "preferredId": 1, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 1, "uri": "urn:ietf:params:rtp-hdrext:sdes:mid"}, {"direction": "sendrecv", "kind": "audio", "preferredEncrypt": false, "preferredId": 4, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 4, "uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 5, "uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"}, {"direction": "sendrecv", "kind": "audio", "preferredEncrypt": false, "preferredId": 10, "uri": "urn:ietf:params:rtp-hdrext:ssrc-audio-level"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 11, "uri": "urn:3gpp:video-orientation"}, {"direction": "sendrecv", "kind": "video", "preferredEncrypt": false, "preferredId": 12, "uri": "urn:ietf:params:rtp-hdrext:toffset"}]}' +1ms
 INFO  mediasoup-client:ReactNativeUnifiedPlan getNativeSctpCapabilities() +39ms
 INFO  mediasoup-client:Device load() | got native SCTP capabilities:'{"numStreams": {"MIS": 1024, "OS": 1024}}' +1ms
 INFO  mediasoup-client:Device load() succeeded +1ms
 INFO  mediasoup-client:ReactNativeUnifiedPlan close() +1ms
 INFO  mediasoup-client:Device createRecvTransport() +0ms
 INFO  mediasoup-client:Transport constructor() [id:a9748ec6-6f55-48bf-9e0d-3087eef4883d, direction:recv] +0ms
 INFO  mediasoup-client:ReactNativeUnifiedPlan run() +2ms
 LOG  rn-webrtc:pc:DEBUG 1 ctor +21ms
 LOG  dtlsParameters: {"fingerprints": [{"algorithm": "sha-256", "value": "7C:72:BC:97:31:9D:ED:20:52:53:71:ED:67:28:91:F4:F2:9C:0E:9F:54:53:06:F1:AF:21:52:AC:B1:8C:F1:E7"}, {"algorithm": "sha-224", "value": "84:36:39:51:23:75:A2:73:D1:85:A5:05:C4:6C:F2:D1:64:0D:F9:65:57:D9:EA:F1:73:F4:F8:98"}, {"algorithm": "sha-384", "value": "5E:71:88:8A:0D:C1:98:88:73:40:DA:16:DB:C8:89:0B:40:DE:2A:DB:85:C3:85:4B:1E:DA:29:DF:0A:83:36:DB:3C:34:4B:3C:EB:7B:05:9B:5B:C0:DA:C8:B8:54:A6:ED"}, {"algorithm": "sha-512", "value": "7C:2F:87:5A:8C:3E:2E:D2:C6:CC:2E:31:69:45:27:46:B5:F9:F4:F9:FA:57:E2:C6:8D:6F:01:F3:7F:DB:76:FD:72:D7:18:71:CF:F6:A3:23:98:DE:EE:63:A3:D6:CB:AD:C2:C4:04:AE:93:4B:16:DE:D2:FA:A8:3B:67:2A:A0:A5"}, {"algorithm": "sha-1", "value": "4B:7F:AE:70:0F:B6:1E:96:71:93:3A:9E:2E:4F:C4:25:80:B2:93:85"}], "role": "auto"}

Any help would be greatly appreciated!

2

Look at the description of the required fields in the client-side ConsumerOptions and compare with what you are passing to the transport.consume. This is just a starting point, there is more wrong.

3

Thanks for the prompt answer and pointer! I reviewed the ConsumerOptions and heopefully fixed transport.consume accordingly (see my updated code above).

The problem is that the program never enters in action === 'transport-connected' and again no trace of icestatechange happening.

4

There should be a server-side consumer id in the first place. Just ask youself: where is this server-side consumer? Did you create it?

5

I think I am creating a consumer id on the server side in the block below, when I create the producer. I then send it to the client.

transport.on('dtlsstatechange', async (dtlsState) => {
            console.log(`transport dtlsState changed to: ${dtlsState}`);
            if (dtlsState === 'connected') {
              console.log('DTLS connected, calling transport.connect');
              await transport.connect({ dtlsParameters: data.dtlsParameters });
              const codec = { ...mediasoupOptions.router.mediaCodecs[0], payloadType: 102 };
              const producer = await transport.produce({ kind: 'video', rtpParameters: { codecs: [codec], encodings: [] } });
          
              // Send the correct rtpParameters to the client
              socket.send(JSON.stringify({
                action: 'transport-connected',
                data: {
                  id: producer.id,
                  rtpParameters: producer.rtpParameters
                }
              }));
            }
          });

The client should then consume it on “transport-connected”, however it never receives such a message. The problem is that the server never fires “transport.on(‘dtlsstatechange’, async (dtlsState)”, and I dont understand why :confused:

6

No, server-side consumer is created with a call to transport.consume (server-side transport).

7

Sorry, did my best reviewing the API but I am not sure how to make the next step. And I am thankful for the direction btw.

Now I have drafted this on the server side, but I am stuck:

            console.log(`transport dtlsState changed to: ${dtlsState}`);
            if (dtlsState === 'connected') {
              console.log('DTLS connected, calling transport.connect');
              await transport.connect({ dtlsParameters: data.dtlsParameters });
              const codec = { ...mediasoupOptions.router.mediaCodecs[0], payloadType: 102 };

              const producer = await transport.produce({ kind: 'video', rtpParameters: { codecs: [codec], encodings: [] } });

              const consumer = await transport.consume({ producerId: producer.id, routerRtpCapabilities: router.rtpCapabilities});
          
              // Send the correct rtpParameters to the client
              socket.send(JSON.stringify({
                action: 'transport-connected',
                data: {
                  id: producer.id,
                  rtpParameters: producer.rtpParameters
                }
              }));
            }
          });

How should I use the consumer? Is the socket.send part correct? Also, if `transport.on(‘dtlsstatechange’)~ is not triggered, whatever I do inside this method will still not fix the problem. It is clear that I am missing something else.

8

The only way to proceed is to read the documentation carefully and understand the mediasoup concept in general. For instance, mediasoup clients use separate transports for sending and receiving, and since client transports connect with the server webrtc transports, there should be separate server transports too.

9

I think you are right. I will study the documentation better and come back here once I’ve made some sensible progress.

11

Alright, I dug into both mediasoup’s documentation and examples and I came up with the following logic:

  • the ffmpeg stream is served to opensoup via a plain RTP stream (PlainTransport)
  • the media is then transmitted from mediasoup to the react-native App using WebRTC (WebRtcTransport);

And here is the code to implement such logic:

Server:

const log = require("./logging");
const WebSocket = require('ws');
const fs = require('fs');
const ffmpeg = require('fluent-ffmpeg');
const { PassThrough } = require('stream');
const Mediasoup = require('mediasoup');

const CryptoSuiteMediasoup = "AES_CM_128_HMAC_SHA1_80";

// ----------------------------------------------------------------------------

// OPtions (to be fut in CONFIG later)
// ============
const mediasoupOptions = {
  worker: {
    rtcMinPort: 10000,
    rtcMaxPort: 10100,
  },
  router: {
    "mediaCodecs": [
      {
          "kind": "audio",
          "mimeType": "audio/opus",
          "clockRate": 48000,
          "channels": 2
      },
      {
          "kind": "video",
          "mimeType": "video/VP8",
          "clockRate": 90000,
          "parameters": {

          }
      },
      {
          "kind": "video",
          "mimeType": "video/H264",
          "clockRate": 90000,
          "parameters": {
              "packetization-mode": 1,
              "profile-level-id": "4d0032",
              "level-asymmetry-allowed": 1
          }
      },
{
          "kind": "video",
          "mimeType": "video/H264",
          "clockRate": 90000,
          "parameters": {
              "packetization-mode": 1,
              "profile-level-id": "42e01f",
              "level-asymmetry-allowed": 1
          }
      }
    ],
  },
};

// ----------------------------------------------------------------------------
// WebSocket server
// ----------------------------------------------------------------------------
const socket = new WebSocket.Server({ port: 8080 });

socket.on('connection', async (socket) => {
  console.log('Client connected');

    socket.on('message', (message) => {
      const parsedMessage = JSON.parse(message);
      log("parsedMessage: ", parsedMessage)
      handleRequest(socket, parsedMessage);
    });
});


// ----------------------------------------------------------------------------

async function handleRequest(socket, request) {
  let responseData = null;

  switch (request.type) {
    case "START_MEDIASOUP": 
      responseData = await handleStartMediasoup();
      break;
    case "WEBRTC_SEND_START": 
      responseData = await handleWebrtcSendStart();
      break;
    case "WEBRTC_SEND_CONNECT":
      responseData = await handleWebrtcSendConnect(request.dtlsParameters);
      break;
    case "WEBRTC_SEND_CONSUME": 
      responseData = await handleWebrtcSendConsume(request.rtpCaps);
      break;
    case "START_VIDEO_RTP_PRODUCER": 
      responseData = await handleStartVideoRtpProducer(request.enableSrtp);
      break;
    case "DEBUG": 
      responseData = await TODO();
      break;
    default:
      log.warn("[handleRequest] Invalid request type:", request.type);
      break;
  }

  socket.send(JSON.stringify({ requestId: request.requestId, type: request.type, data: responseData }));
}


// ----------------------------------------------------------------------------

let router;
let worker;
let transport;
let videoProducer;

// ----------------------------------------------------------------------------

// Creates a mediasoup worker and router

async function handleStartMediasoup() {
  log(0)
  
  try {
    worker = await Mediasoup.createWorker(mediasoupOptions.worker);
  } catch (err) {
    log.error("[handleStartMediasoup] ERROR:", err);
    process.exit(1);
  }

  worker.on("died", () => {
    log.error(
      "mediasoup worker died, exit in 3 seconds... [pid:%d]",
      worker.pid
    );
    setTimeout(() => process.exit(1), 3000);
  });

  log("[handleStartMediasoup] mediasoup worker created [pid:%d]", worker.pid);

  
  try {
    router = await worker.createRouter(mediasoupOptions.router);
  } catch (err) {
    log.error("[handleStartMediasoup] ERROR:", err);
    process.exit(1);
  }

  // At this point, the computed "router.rtpCapabilities" includes the
  // router codecs enhanced with retransmission and RTCP capabilities,
  // and the list of RTP header extensions supported by mediasoup.

  log("[handleStartMediasoup] mediasoup router created");
  log.trace(
    "[handleStartMediasoup] mediasoup router RtpCapabilities:\n%O",
    router.rtpCapabilities
  );

  return router.rtpCapabilities;

}


// ----------------------------------------------------------------------------
// WebRtcTransport Consumer
// ----------------------------------------------------------------------------

// Creates a mediasoup WebRTC SEND transport

async function handleWebrtcSendStart() {
  
  try {
    transport = await router.createWebRtcTransport({
      // webRtcServer: webrtc_server,
      listenIps: [{ ip: "127.0.0.1", announcedIp: null }],
      enableSctp: true,
      enableUdp: true,
      enableTcp: true,
      preferUdp: true,
      initialAvailableOutgoingBitrate: 300000,
  });
  } catch (err) {
    log.error("[handleWebrtcSendStart] ERROR:", err);
    process.exit(1);
  }

  log("[handleWebrtcSendStart] mediasoup WebRTC SEND transport created");

  const webrtcTransportOptions = {
    id: transport.id,
    iceParameters: transport.iceParameters,
    iceCandidates: transport.iceCandidates,
    dtlsParameters: transport.dtlsParameters,
    sctpParameters: transport.sctpParameters,
  };

  log.trace(
    "[handleWebrtcSendStart] mediasoup WebRTC SEND TransportOptions:\n%O",
    webrtcTransportOptions
  );

  return webrtcTransportOptions;
}

// ----------------------------------------------------------------------------

// Calls WebRtcTransport.connect() whenever ???

async function handleWebrtcSendConnect(dtlsParams) {

  await transport.connect({ dtlsParameters: dtlsParams });

  log("[handleWebrtcSendConnect] mediasoup WebRTC SEND transport connected");
}

// Calls WebRtcTransport.consume() to start sending media to the browser

async function handleWebrtcSendConsume(rtpCaps) {
  
  if (!videoProducer) {
    log.error("[handleWebrtcSendConsume] BUG: The videoProducer doesn't exist!");
    process.exit(1);
  }

  const consumer = await transport.consume({
    producerId: videoProducer.id,
    rtpCapabilities: rtpCaps,
    paused: true,
  });

  log(
    "[handleWebrtcSendConsume] mediasoup WebRTC SEND consumer created, kind: %s, type: %s, paused: %s",
    consumer.kind,
    consumer.type,
    consumer.paused
  );

  log.trace(
    "[handleWebrtcSendConsume] mediasoup WebRTC SEND consumer RtpParameters:\n%O",
    consumer.rtpParameters
  );

  const webrtcConsumerOptions = {
    id: consumer.id,
    producerId: consumer.producerId,
    kind: consumer.kind,
    rtpParameters: consumer.rtpParameters,
  };

  return webrtcConsumerOptions;
}

// ----------------------------------------------------------------------------
// PlainRtpTransport Producer
// ----------------------------------------------------------------------------

async function handleStartVideoRtpProducer(enableSrtp) {
  const videoTransport = await router.createPlainTransport(
    { 
      listenIp : '127.0.0.1',
      rtcpMux  : false,
      comedia  : true,

      // Enable SRTP if requested
      enableSrtp: enableSrtp,
      srtpCryptoSuite: CryptoSuiteMediasoup,
    });
  
  // Read the transport local RTP port.
  // const videoRtpPort = videoTransport.tuple.localPort;
  // // => 
  
  // // Read the transport local RTCP port.
  // const videoRtcpPort = videoTransport.rtcpTuple.localPort;
  // // => 

  log(
    "[startVideoRtpProducer] mediasoup RTP RECV transport created: %s:%d (%s)",
    videoTransport.tuple.localIp,
    videoTransport.tuple.localPort,
    videoTransport.tuple.protocol
  );

  log(
    "[startVideoRtpProducer] mediasoup RTCP RECV transport created: %s:%d (%s)",
    videoTransport.rtcpTuple.localIp,
    videoTransport.rtcpTuple.localPort,
    videoTransport.rtcpTuple.protocol
  );

  // COMEDIA is enabled, so the transport connection will happen asynchronously

  videoTransport.on("tuple", (tuple) => {
    log(
      "[startVideoRtpProducer] mediasoup RTP RECV transport connected: %s:%d <--> %s:%d (%s)",
      tuple.localIp,
      tuple.localPort,
      tuple.remoteIp,
      tuple.remotePort,
      tuple.protocol
    );
  });

  videoTransport.on("rtcptuple", (rtcpTuple) => {
    log(
      "[startVideoRtpProducer] mediasoup RTCP RECV transport connected: %s:%d <--> %s:%d (%s)",
      rtcpTuple.localIp,
      rtcpTuple.localPort,
      rtcpTuple.remoteIp,
      rtcpTuple.remotePort,
      rtcpTuple.protocol
    );
  });

  // mediasoup RTP producer (receive media from ffmpeg stream)
  // ---------------------------------------------------

  try {
    videoProducer = await videoTransport.produce(
      {
        kind          : 'video',
        paused: false,
        rtpParameters :
        {
          codecs :
          [
            {
              mimeType     : 'video/vp8',
              clockRate    : 90000,
              payloadType  : 102,
              rtcpFeedback : [ ], // FFmpeg does not support NACK nor PLI/FIR.
            }
          ],
          encodings : [ { ssrc: 22222222 } ]
        }
      });
  } catch (err) {
    log.error("[startVideoRtpProducer] ERROR:", err);
    process.exit(1);
  }

  log(
    "[startVideoRtpProducer] mediasoup RTP RECV producer created, kind: %s, type: %s, paused: %s",
    videoProducer.kind,
    videoProducer.type,
    videoProducer.paused
  );

  log(
    "[startVideoRtpProducer] mediasoup RTP RECV producer RtpParameters:\n%O",
    videoProducer.rtpParameters
  );

  // Connect the mediasoup transport to enable receiving (S)RTP/RTCP and sending
  // (S)RTCP packets
 // IMPORTANT: if comedia is enabled in this plain transport and SRTP is not, connect() must not be called.
  let srtpParams = undefined;
  if (enableSrtp) {
    srtpParams = {
      cryptoSuite: CryptoSuiteMediasoup,
      keyBase64: CONFIG.srtp.keyBase64,
    };

    await videoTransport.connect({
      srtpParameters: srtpParams,
    });
    log("[startVideoRtpProducer] mediasoup RTP RECV producer connected");
  } 

}


// ----------------------------------------------------------------------------



console.log('Server listening on port 8080');

client

import React, { useEffect, useState } from 'react';
import { StyleSheet, View } from 'react-native';
const MediasoupClient = require("mediasoup-client");
import { registerGlobals } from 'react-native-webrtc';
import { RTCView } from 'react-native-webrtc';



// new stuff
const log = require("./logging");

//

registerGlobals(); // Register WebRTC globals



// ----------------------------------------------------------------------------

// Utility functions
// ================

let requestId = 0;
const pendingRequests = new Map();

function socketRequest(socket, message) {
  return new Promise((resolve, reject) => {
    const currentRequestId = requestId++;
    message.requestId = currentRequestId;

    pendingRequests.set(currentRequestId, { resolve, reject });
    socket.send(JSON.stringify(message));
  });
}


// ----------------------------------------------------------------------------



socket = new WebSocket('ws://192.168.122.1:8080');



const App = () => {
  // const [socket, setSocket] = useState(null);
  
  const [videoTrack, setVideoTrack] = useState(null);

  WebSocket.prototype.sendAsync = function (data) {
    return new Promise((resolve, reject) => {
      this.send(JSON.stringify(data), (err) => {
        if (err) {
          reject(err);
        } else {
          resolve();
        }
      });
    });
  };

  let device;

  useEffect(() => {

    // Setup a single 'onmessage' event listener for the socket
    // This implementation assigns a unique ID to each request, and it 
    // stores the pending requests in a pendingRequests Map. When a response 
    // is received, it looks up the corresponding request by its ID and 
    // resolves or rejects the Promise accordingly.
    socket.onmessage = (event) => {
      const response = JSON.parse(event.data);
      const { requestId, error } = response;

      const pendingRequest = pendingRequests.get(requestId);

      if (pendingRequest) {
        const { resolve, reject } = pendingRequest;

        if (error) {
          reject(error);
        } else {
          resolve(response);
        }

        pendingRequests.delete(requestId);
      } else {
        console.error('Received response for unknown request ID:', requestId);
      }
    };

    socket.onopen = async () => {
      console.log('Connected to server');
      await startWebRTC()
    };

    async function startWebRTC() {
      log("[startWebRTC] Start WebRTC transmission from browser to mediasoup");
      await startMediasoup();
      await startVideoRtpProducer();
      await startWebrtcRecv();
    }


    async function startMediasoup() {
    
      const response = await socketRequest(socket, { type: "START_MEDIASOUP" });
      const routerRtpCaps = response.data;

      log("[startMediasoup] mediasoup router created");
    

      try {
        device = new MediasoupClient.Device();
      } catch (err) {
        log.error("[startMediasoup] ERROR:", err);
        return;
      }
    
      try {
        await device.load({ routerRtpCapabilities: routerRtpCaps });
      } catch (err) {
        log.error("[startMediasoup] ERROR:", err);
        return;
      }
    
      log(
        "[startMediasoup] mediasoup device created, handlerName: %s, use audio: %s, use video: %s",
        device.handlerName,
        device.canProduce("audio"),
        device.canProduce("video")
      );
    
      log.trace(
        "[startMediasoup] Device RtpCapabilities:\n%O",
        device.rtpCapabilities
      );
    }


    // ----

    async function startVideoRtpProducer() {
    
      // start VideoRtpProducer
      // --------------------------

      // TODO: put this in config and allow for encryption too
      let enableSrtp = false;
      const response = await socketRequest(socket, {
        type: "START_VIDEO_RTP_PRODUCER",
        enableSrtp: enableSrtp,
      });

      log("[startVideoRtpProducer] VideoRtpProducer consumer created");

    }

    async function startWebrtcRecv() {
    
      // mediasoup WebRTC transport
      // --------------------------

      let response = await socketRequest(socket, { type: "WEBRTC_SEND_START" });
      const webrtcTransportOptions = response.data;
    
      log("[startWebrtcRecv] WebRTC SEND transport created");
    
      let transport;
      try {
        transport = device.createRecvTransport(webrtcTransportOptions);
      } catch (err) {
        log.error("[startWebrtcRecv] ERROR:", err);
        return;
      }
    
      log("[startWebrtcRecv] WebRTC RECV transport created");
    
      // "connect" is emitted upon the first call to transport.consume()

      try {
        transport.on("connect", ({ dtlsParameters }, _errback) => {
          // Signal local DTLS parameters to the server side transport
          response = socketRequest(socket, {
            type: "WEBRTC_SEND_CONNECT",
            dtlsParameters: dtlsParameters,
          })
        });
      } catch (err) {
        log.error("[startWebrtcRecv] ERROR:", err);
        return;
       }

       log(device.rtpCapabilities)
    
      // mediasoup WebRTC consumer
      // -------------------------
      try {
        response = await socketRequest(socket, {
        type: "WEBRTC_SEND_CONSUME",
        rtpCaps: device.rtpCapabilities,
       });
      } catch (err) {
        log.error("[startWebrtcRecv] ERROR:", err);
        return;
       }


      
      const webrtcConsumerOptions = response.data;
    
      log("[startWebrtcRecv] WebRTC SEND consumer created");
    
      let useVideo = true;

      log(11111)
    
      // Start mediasoup-client's WebRTC consumer(s)
    
      const stream = new MediaStream();    

      log(22222)
    
      if (useVideo) {
        log(33333)
        log("webrtcConsumerOptions: ", webrtcConsumerOptions)
        const consumer = await transport.consume(webrtcConsumerOptions);
        log(44444)
        stream.addTrack(consumer.track);
        log(55555)
        log("[startWebrtcRecv] WebRTC RECV consumer created");

        // Update the videoTrack state
        setVideoTrack(consumer.track);
      }
      
    }
        
  }, []);

  return (
    <View style={styles.container}>
      {videoTrack && (
        <RTCView
          style={styles.video}
          streamURL={videoTrack.toURL()}
          objectFit="cover"
        />
      )}
    </View>
  );
};

const styles = StyleSheet.create({
  container: {
    flex: 1,
    justifyContent: 'center',
    alignItems: 'center',
    backgroundColor: '#F5FCFF',
  },
  video: {
    width: '100%',
    height: '100%',
  },
});

export default App;

ffmpeg command

ffmpeg    -re     -v info         -stream_loop -1         -i ./data/example.mp4   -map 0:v:0      -pix_fmt yuv420p -c:v libvpx -b:v 1000k -deadline realtime -cpu-used 4     -f tee  "[select=v:f=rtp:ssrc=22222222:payload_type=102]rtp://127.0.0.1:10024?rtcpport=10025"

And here are the logs.

Server:

  demo:info parsedMessage:  { type: 'START_MEDIASOUP', requestId: 0 } +0ms
  demo:info 0 +1ms
  demo:info [handleStartMediasoup] mediasoup worker created [pid:217285] +9ms
  demo:info [handleStartMediasoup] mediasoup router created +2ms
  demo:info parsedMessage:  { type: 'START_VIDEO_RTP_PRODUCER', enableSrtp: false, requestId: 1 } +89ms
  demo:info [startVideoRtpProducer] mediasoup RTP RECV transport created: 127.0.0.1:10024 (udp) +2ms
  demo:info [startVideoRtpProducer] mediasoup RTCP RECV transport created: 127.0.0.1:10025 (udp) +0ms
  demo:info [startVideoRtpProducer] mediasoup RTP RECV producer created, kind: video, type: simple, paused: false +1ms
  demo:info [startVideoRtpProducer] mediasoup RTP RECV producer RtpParameters:
  demo:info {
  demo:info   codecs: [
  demo:info     {
  demo:info       mimeType: 'video/vp8',
  demo:info       clockRate: 90000,
  demo:info       payloadType: 102,
  demo:info       rtcpFeedback: [],
  demo:info       parameters: {}
  demo:info     }
  demo:info   ],
  demo:info   encodings: [ { ssrc: 22222222, dtx: false } ],
  demo:info   headerExtensions: [],
  demo:info   rtcp: { reducedSize: true, cname: 'be692c07' }
  demo:info } +1ms
  demo:info [startVideoRtpProducer] mediasoup RTP RECV producer connected +0ms
  demo:info parsedMessage:  { type: 'WEBRTC_SEND_START', requestId: 2 } +3ms
  demo:info [handleWebrtcSendStart] mediasoup WebRTC SEND transport created +1ms
  demo:info parsedMessage:  {
  type: 'WEBRTC_SEND_CONSUME',
  rtpCaps: {
    codecs: [ [Object], [Object], [Object] ],
    headerExtensions: [
      [Object], [Object],
      [Object], [Object],
      [Object], [Object],
      [Object], [Object]
    ]
  },
  requestId: 3
} +15ms
  demo:info [handleWebrtcSendConsume] mediasoup WebRTC SEND consumer created, kind: video, type: simple, paused: true +2ms
  demo:info parsedMessage:  {
  type: 'WEBRTC_SEND_CONNECT',
  dtlsParameters: { role: 'client', fingerprints: [ [Object] ] },
  requestId: 4
} +35ms
  demo:info [handleWebrtcSendConnect] mediasoup WebRTC SEND transport connected +0ms
  demo:info [startVideoRtpProducer] mediasoup RTCP RECV transport connected: 127.0.0.1:10025 <--> 127.0.0.1:47132 (udp) +13s
  demo:info [startVideoRtpProducer] mediasoup RTP RECV transport connected: 127.0.0.1:10024 <--> 127.0.0.1:47131 (udp) +0ms

Client

 DEBUG  demo:info [startWebrtcRecv] WebRTC SEND consumer created +7ms
 DEBUG  demo:info '11111' +0ms
 DEBUG  demo:info '22222' +0ms
 DEBUG  demo:info '33333' +0ms
 DEBUG  demo:info webrtcConsumerOptions:  +0ms {"id": "8d29a35c-26b0-48d3-9943-e04a01fc4108", "kind": "video", "producerId": "86a30015-7a63-4598-801d-ed1f9b9712a8", "rtpParameters": {"codecs": [[Object], [Object]], "encodings": [[Object]], "headerExtensions": [[Object], [Object], [Object], [Object], [Object]], "mid": "0", "rtcp": {"cname": "be692c07", "mux": true, "reducedSize": true}}}
 INFO  mediasoup-client:Transport consume() +17ms
 INFO  awaitqueue push() [name:transport.createPendingConsumers()] +0ms
 INFO  awaitqueue execute() [name:transport.createPendingConsumers()] +1ms
 INFO  mediasoup-client:ReactNativeUnifiedPlan receive() [trackId:8d29a35c-26b0-48d3-9943-e04a01fc4108, kind:video] +19ms
 INFO  mediasoup-client:ReactNativeUnifiedPlan receive() | calling pc.setRemoteDescription() [offer:'{"sdp": "v=0\r\n' +
  'o=mediasoup-client 10000 1 IN IP4 0.0.0.0\r\n' +
  's=-\r\n' +
  't=0 0\r\n' +
  'a=ice-lite\r\n' +
  'a=fingerprint:sha-1 E0:C2:79:3C:B3:28:DC:C4:25:84:E8:E5:DC:46:38:07:FA:22:A7:92\r\n' +
  'a=msid-semantic: WMS *\r\n' +
  'a=group:BUNDLE 0\r\n' +
  'm=video 7 UDP/TLS/RTP/SAVPF 101 102\r\n' +
  'c=IN IP4 127.0.0.1\r\n' +
  'a=rtpmap:101 VP8/90000\r\n' +
  'a=rtpmap:102 rtx/90000\r\n' +
  'a=fmtp:102 apt=101\r\n' +
  'a=rtcp-fb:101 transport-cc \r\n' +
  'a=rtcp-fb:101 ccm fir\r\n' +
  'a=rtcp-fb:101 nack \r\n' +
  'a=rtcp-fb:101 nack pli\r\n' +
  'a=extmap:1 urn:ietf:params:rtp-hdrext:sdes:mid\r\n' +
  'a=extmap:4 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n' +
  'a=extmap:5 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n' +
  'a=extmap:11 urn:3gpp:video-orientation\r\n' +
  'a=extmap:12 urn:ietf:params:rtp-hdrext:toffset\r\n' +
  'a=setup:actpass\r\n' +
  'a=mid:0\r\n' +
  'a=msid:be692c07 8d29a35c-26b0-48d3-9943-e04a01fc4108\r\n' +
  'a=sendonly\r\n' +
  'a=ice-ufrag:uyixrilzwysexuf8nn72gje01rhfq7wh\r\n' +
  'a=ice-pwd:44sz18zdaciovw9qxhx8at4a3drpsta3\r\n' +
  'a=candidate:udpcandidate 1 udp 1076558079 127.0.0.1 10065 typ host\r\n' +
  'a=candidate:tcpcandidate 1 tcp 1076302079 127.0.0.1 10083 typ host tcptype passive\r\n' +
  'a=end-of-candidates\r\n' +
  'a=ice-options:renomination\r\n' +
  'a=ssrc:316318477 cname:be692c07\r\n' +
  'a=ssrc:316318478 cname:be692c07\r\n' +
  'a=ssrc-group:FID 316318477 316318478\r\n' +
  'a=rtcp-mux\r\n' +
  'a=rtcp-rsize\r\n' +
  '", "type": "offer"}'] +5ms
 LOG  rn-webrtc:pc:DEBUG 1 setRemoteDescription +20ms
 LOG  rn-webrtc:pc:DEBUG 1 ontrack +10ms
 LOG  rn-webrtc:pc:DEBUG 1 setRemoteDescription OK +4ms
 LOG  rn-webrtc:pc:DEBUG 1 createAnswer +0ms
 INFO  mediasoup-client:RemoteSdp updateDtlsRole() [role:server] +0ms

Assumed that the program logic is correct, the issue I am having is at the very last step, The client is stuck on

const consumer = await transport.consume(webrtcConsumerOptions);

And never goes to

stream.addTrack(consumer.track);
        log("[startWebrtcRecv] WebRTC RECV consumer created");

        // Update the videoTrack state
        setVideoTrack(consumer.track);
      }

I don’t of course expect help in debugging every little detail, but I would really appreciate a high-level feedback about whether I am on the right path and if I am making some evident mistakes. Thank you!

12

Issue solved!

For the books, it was a missing callback() in the client here:

try {
        transport.on("connect", ({ dtlsParameters }, callback, _errback) => {
          // Signal local DTLS parameters to the server side transport
          response = socketRequest(socket, {
            type: "WEBRTC_SEND_CONNECT",
            dtlsParameters: dtlsParameters,
          })
          callback();
        });
      } catch (err) {
        log.error("[startWebrtcRecv] ERROR:", err);
        return;
       }
2 Likes