Hello,
First congrats for MediaSoup which is a truly amazing SFU!
Here is the case: Gstreamer sends 3 RTP H264 simulcast streams to MediaSoup.
Once the Consumer selects one of the 3 streams, it never upgrades nor downgrades to the next simulcast stream. If the bandwidth gets low and the Consumer is on a mid/high stream, the video freezes but the layer is not switched.
Producer (PlainTransport):
{
"id": "811d97f1-24e1-4076-8af0-6b880db330ae",
"kind": "video",
"paused": false,
"rtpMapping": {
"codecs": [
{
"mappedPayloadType": 101,
"payloadType": 101
},
{
"mappedPayloadType": 102,
"payloadType": 102
}
],
"encodings": [
{
"mappedSsrc": 590088931,
"rid": null,
"ssrc": 5555
},
{
"mappedSsrc": 590088932,
"rid": null,
"ssrc": 6666
},
{
"mappedSsrc": 590088933,
"rid": null,
"ssrc": 7777
}
]
},
"rtpParameters": {
"codecs": [
{
"clockRate": 90000,
"mimeType": "video/H264",
"parameters": {
"level-asymmetry-allowed": 1,
"packetization-mode": 1,
"profile-level-id": "42e01f"
},
"payloadType": 101,
"rtcpFeedback": [
{
"type": "nack"
},
{
"parameter": "pli",
"type": "nack"
},
{
"parameter": "fir",
"type": "ccm"
}
]
},
{
"clockRate": 90000,
"mimeType": "video/rtx",
"parameters": {
"apt": 101
},
"payloadType": 102,
"rtcpFeedback": []
}
],
"encodings": [
{
"codecPayloadType": 101,
"ksvc": false,
"maxFramerate": 15,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 5555,
"temporalLayers": 1
},
{
"codecPayloadType": 101,
"ksvc": false,
"maxFramerate": 30,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 6666,
"temporalLayers": 1
},
{
"codecPayloadType": 101,
"ksvc": false,
"maxFramerate": 30,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 7777,
"temporalLayers": 1
}
],
"headerExtensions": [
{
"encrypt": false,
"id": 4,
"parameters": {},
"uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"
},
{
"encrypt": false,
"id": 11,
"parameters": {},
"uri": "urn:3gpp:video-orientation"
},
{
"encrypt": false,
"id": 12,
"parameters": {},
"uri": "urn:ietf:params:rtp-hdrext:toffset"
}
],
"rtcp": {
"cname": "CN1TkXf4Iw",
"reducedSize": true
}
},
"rtpStreams": [
{
"params": {
"clockRate": 90000,
"cname": "CN1TkXf4Iw",
"encodingIdx": 0,
"mimeType": "video/H264",
"payloadType": 101,
"spatialLayers": 1,
"ssrc": 5555,
"temporalLayers": 1,
"useDtx": false,
"useFir": true,
"useInBandFec": false,
"useNack": true,
"usePli": true
},
"score": 10
},
{
"params": {
"clockRate": 90000,
"cname": "CN1TkXf4Iw",
"encodingIdx": 1,
"mimeType": "video/H264",
"payloadType": 101,
"spatialLayers": 1,
"ssrc": 6666,
"temporalLayers": 1,
"useDtx": false,
"useFir": true,
"useInBandFec": false,
"useNack": true,
"usePli": true
},
"score": 10
},
{
"params": {
"clockRate": 90000,
"cname": "CN1TkXf4Iw",
"encodingIdx": 2,
"mimeType": "video/H264",
"payloadType": 101,
"spatialLayers": 1,
"ssrc": 7777,
"temporalLayers": 1,
"useDtx": false,
"useFir": true,
"useInBandFec": false,
"useNack": true,
"usePli": true
},
"score": 10
}
],
"traceEventTypes": "keyframe,nack,pli,fir",
"type": "simulcast"
}
WebRTC Consumer (talking to Chrome), right after creation:
{
"consumableRtpEncodings": [
{
"ksvc": false,
"maxFramerate": 15,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 590088931,
"temporalLayers": 1
},
{
"ksvc": false,
"maxFramerate": 30,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 590088932,
"temporalLayers": 1
},
{
"ksvc": false,
"maxFramerate": 30,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 590088933,
"temporalLayers": 1
}
],
"currentSpatialLayer": -1,
"currentTemporalLayer": -1,
"id": "a81a58e1-a285-476c-80f4-d9f30f64fe41",
"kind": "video",
"paused": true,
"preferredSpatialLayer": 2,
"preferredTemporalLayer": 0,
"priority": 1,
"producerId": "811d97f1-24e1-4076-8af0-6b880db330ae",
"producerPaused": false,
"rtpParameters": {
"codecs": [
{
"clockRate": 90000,
"mimeType": "video/H264",
"parameters": {
"level-asymmetry-allowed": 1,
"packetization-mode": 1,
"profile-level-id": "42e01f"
},
"payloadType": 101,
"rtcpFeedback": [
{
"type": "transport-cc"
},
{
"parameter": "fir",
"type": "ccm"
},
{
"type": "nack"
},
{
"parameter": "pli",
"type": "nack"
}
]
},
{
"clockRate": 90000,
"mimeType": "video/rtx",
"parameters": {
"apt": 101
},
"payloadType": 102,
"rtcpFeedback": []
}
],
"encodings": [
{
"codecPayloadType": 101,
"ksvc": false,
"rtx": {
"ssrc": 867747230
},
"scalabilityMode": "S3T1",
"spatialLayers": 3,
"ssrc": 867747229,
"temporalLayers": 1
}
],
"headerExtensions": [
{
"encrypt": false,
"id": 1,
"parameters": {},
"uri": "urn:ietf:params:rtp-hdrext:sdes:mid"
},
{
"encrypt": false,
"id": 4,
"parameters": {},
"uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"
},
{
"encrypt": false,
"id": 5,
"parameters": {},
"uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"
},
{
"encrypt": false,
"id": 11,
"parameters": {},
"uri": "urn:3gpp:video-orientation"
},
{
"encrypt": false,
"id": 12,
"parameters": {},
"uri": "urn:ietf:params:rtp-hdrext:toffset"
}
],
"mid": "1",
"rtcp": {
"cname": "CN1TkXf4Iw",
"reducedSize": true
}
},
"rtpStream": {
"params": {
"clockRate": 90000,
"cname": "CN1TkXf4Iw",
"encodingIdx": 0,
"mimeType": "video/H264",
"payloadType": 101,
"rtxPayloadType": 102,
"rtxSsrc": 867747230,
"spatialLayers": 3,
"ssrc": 867747229,
"temporalLayers": 1,
"useDtx": false,
"useFir": true,
"useInBandFec": false,
"useNack": true,
"usePli": true
},
"rtxStream": {
"params": {
"clockRate": 90000,
"cname": "CN1TkXf4Iw",
"mimeType": "video/rtx",
"payloadType": 102,
"ssrc": 867747230
}
},
"score": 10
},
"supportedCodecPayloadTypes": [
101
],
"targetSpatialLayer": -1,
"targetTemporalLayer": -1,
"traceEventTypes": "keyframe,nack,pli,fir",
"type": "simulcast"
}
Same Consumer 30 seconds later:
{
"consumableRtpEncodings": [
{
"ksvc": false,
"maxFramerate": 15,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 590088931,
"temporalLayers": 1
},
{
"ksvc": false,
"maxFramerate": 30,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 590088932,
"temporalLayers": 1
},
{
"ksvc": false,
"maxFramerate": 30,
"scalabilityMode": "S1T1",
"spatialLayers": 1,
"ssrc": 590088933,
"temporalLayers": 1
}
],
"currentSpatialLayer": 0,
"currentTemporalLayer": 0,
"id": "a81a58e1-a285-476c-80f4-d9f30f64fe41",
"kind": "video",
"paused": false,
"preferredSpatialLayer": 2,
"preferredTemporalLayer": 0,
"priority": 1,
"producerId": "811d97f1-24e1-4076-8af0-6b880db330ae",
"producerPaused": false,
"rtpParameters": {
"codecs": [
{
"clockRate": 90000,
"mimeType": "video/H264",
"parameters": {
"level-asymmetry-allowed": 1,
"packetization-mode": 1,
"profile-level-id": "42e01f"
},
"payloadType": 101,
"rtcpFeedback": [
{
"type": "transport-cc"
},
{
"parameter": "fir",
"type": "ccm"
},
{
"type": "nack"
},
{
"parameter": "pli",
"type": "nack"
}
]
},
{
"clockRate": 90000,
"mimeType": "video/rtx",
"parameters": {
"apt": 101
},
"payloadType": 102,
"rtcpFeedback": []
}
],
"encodings": [
{
"codecPayloadType": 101,
"ksvc": false,
"rtx": {
"ssrc": 867747230
},
"scalabilityMode": "S3T1",
"spatialLayers": 3,
"ssrc": 867747229,
"temporalLayers": 1
}
],
"headerExtensions": [
{
"encrypt": false,
"id": 1,
"parameters": {},
"uri": "urn:ietf:params:rtp-hdrext:sdes:mid"
},
{
"encrypt": false,
"id": 4,
"parameters": {},
"uri": "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time"
},
{
"encrypt": false,
"id": 5,
"parameters": {},
"uri": "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01"
},
{
"encrypt": false,
"id": 11,
"parameters": {},
"uri": "urn:3gpp:video-orientation"
},
{
"encrypt": false,
"id": 12,
"parameters": {},
"uri": "urn:ietf:params:rtp-hdrext:toffset"
}
],
"mid": "1",
"rtcp": {
"cname": "CN1TkXf4Iw",
"reducedSize": true
}
},
"rtpStream": {
"params": {
"clockRate": 90000,
"cname": "CN1TkXf4Iw",
"encodingIdx": 0,
"mimeType": "video/H264",
"payloadType": 101,
"rtxPayloadType": 102,
"rtxSsrc": 867747230,
"spatialLayers": 3,
"ssrc": 867747229,
"temporalLayers": 1,
"useDtx": false,
"useFir": true,
"useInBandFec": false,
"useNack": true,
"usePli": true
},
"rtxStream": {
"params": {
"clockRate": 90000,
"cname": "CN1TkXf4Iw",
"mimeType": "video/rtx",
"payloadType": 102,
"ssrc": 867747230
}
},
"score": 10
},
"supportedCodecPayloadTypes": [
101
],
"targetSpatialLayer": 0,
"targetTemporalLayer": 0,
"traceEventTypes": "keyframe,nack,pli,fir",
"type": "simulcast"
}
and the Consumer stats:
[
{
bitrate: 515917,
byteCount: 1825786,
firCount: 0,
fractionLost: 0,
kind: 'video',
mimeType: 'video/H264',
nackCount: 0,
nackPacketCount: 0,
packetCount: 2303,
packetsDiscarded: 0,
packetsLost: 0,
packetsRepaired: 0,
packetsRetransmitted: 0,
pliCount: 0,
roundTripTime: 78.338623046875,
rtxPacketsDiscarded: 0,
rtxSsrc: 867747230,
score: 10,
ssrc: 867747229,
timestamp: 76676578,
type: 'outbound-rtp'
},
{
bitrate: 503373,
byteCount: 1779740,
firCount: 0,
fractionLost: 0,
jitter: 0,
kind: 'video',
mimeType: 'video/H264',
nackCount: 0,
nackPacketCount: 0,
packetCount: 2304,
packetsDiscarded: 0,
packetsLost: 0,
packetsRepaired: 0,
packetsRetransmitted: 0,
pliCount: 1,
score: 10,
ssrc: 5555,
timestamp: 76676578,
type: 'inbound-rtp'
}
]
PLI & Nack between Gstreamer and MediaSoup seem to work correctly, as MediaSoup logs show:
[02:38:17.859] producer "score" event [producerId:811d97f1-24e1-4076-8af0-6b880db330ae, score:[ { encodingIdx: 0, score: 10, ssrc: 5555 } ]]
[02:38:17.859] producer trace event [producerId:811d97f1-24e1-4076-8af0-6b880db330ae, msg.type:pli, msg:{ direction: 'out', info: { ssrc: 5555 }, timestamp: 76648547, type: 'pli' }]
[02:38:17.859] producer trace event [producerId:811d97f1-24e1-4076-8af0-6b880db330ae, msg.type:nack, msg:{ direction: 'out', info: {}, timestamp: 76648547, type: 'nack' }]
[02:38:17.860] producer trace event [producerId:811d97f1-24e1-4076-8af0-6b880db330ae, msg.type:keyframe, msg:{ direction: 'in', info: { isKeyFrame: true, marker: 'false', payloadSize: 28, payloadType: 101, sequenceNumber: 19740, size:
40, spatialLayer: 0, ssrc: 5555, temporalLayer: 0, timestamp: 3512210647 }, timestamp: 76648547, type: 'keyframe' }]
[02:38:17.860] Consumer _layerschange_ event [spatialLayer: 0, temporalLayer: 0, consumerId: a81a58e1-a285-476c-80f4-d9f30f64fe41]
[02:38:17.860] Consumer trace event [consumerId:a81a58e1-a285-476c-80f4-d9f30f64fe41, msg.type:keyframe, msg:{ direction: 'out', info: { isKeyFrame: true, marker: 'false', mid: '1', payloadSize: 28, payloadType: 101, sequenceNumber: 1, size: 60, spatialLayer: 0, ssrc: 867747229, temporalLayer: 0, timestamp: 3512210647, wideSequenceNumber: 17 }, timestamp: 76648547, type: 'keyframe' }]
However when the time comes to switch to a higher bitrate stream:
[02:38:23.868] Consumer Transport-WebRTC trace event [transportId:c036bd68-c4f7-4666-83d9-9083a3c20230, msg.type:bwe, msg:{ direction: 'out', info: { availableBitrate: 2139400, desiredBitrate: 2107271, effectiveDesiredBitrate: 2268809, maxBitrate: 3062892, maxPaddingBitrate: 2603458, minBitrate: 30000, startBitrate: 2139400, type: 'transport-cc' }, timestamp: 76654556, type: 'bwe' }]
[02:38:25.869] Consumer Transport-WebRTC trace event [transportId:c036bd68-c4f7-4666-83d9-9083a3c20230, msg.type:bwe, msg:{ direction: 'out', info: { availableBitrate: 2139400, desiredBitrate: 1858310, effectiveDesiredBitrate: 2016565, maxBitrate: 2722362, maxPaddingBitrate: 2314007, minBitrate: 30000, startBitrate: 2139400, type: 'transport-cc' }, timestamp: 76656557, type: 'bwe' }]
nothing happens, despite the keyframes from the other streams:
[02:38:24.204] producer _trace_ event [producerId:811d97f1-24e1-4076-8af0-6b880db330ae, msg.type:keyframe, msg:{ direction: 'in', info: { isKeyFrame: true, marker: 'false', payloadSize: 28, payloadType: 101, sequenceNumber: 15093, size: 40, spatialLayer: 0, ssrc: 6666, temporalLayer: 0, timestamp: 2987869078 }, timestamp: 76654893, type: 'keyframe' }]
[02:38:24.434] producer _trace_ event [producerId:811d97f1-24e1-4076-8af0-6b880db330ae, msg.type:keyframe, msg:{ direction: 'in', info: { isKeyFrame: true, marker: 'false', payloadSize: 28, payloadType: 101, sequenceNumber: 24085, size: 40, spatialLayer: 0, ssrc: 7777, temporalLayer: 0, timestamp: 584614784 }, timestamp: 76655123, type: 'keyframe' }]
If I force a low bandwidth on the laptop with Chrome, I get:
[02:38:48.238] Consumer _layerschange_ event [spatialLayer: -, temporalLayer: -, consumerId: a81a58e1-a285-476c-80f4-d9f30f64fe41]
[02:38:48.238] Consumer Transport-WebRTC trace event [transportId:c036bd68-c4f7-4666-83d9-9083a3c20230, msg.type:bwe, msg:{ direction: 'out', info: { availableBitrate: 489440, desiredBitrate: 1690909, effectiveDesiredBitrate: 1690909, maxBitrate: 2282727, maxPaddingBitrate: 1940318, minBitrate: 30000, startBitrate: 489440, type: 'transport-cc' }, timestamp: 76678926, type: 'bwe' }]
The same happens when the consumer starts on the highest-bitrate stream and then the bandwidth is reduced. The consumer stays on the highest-bitrate instead of downgrading to the mid.
Another example here (different session, so not the same ids) where the availableBitrate is below the highest-bitrate but above the mid-bitrate stream. It should have switched layers but instead we get:
[06:58:53.464] Consumer _layerschange_ event [spatialLayer: -, temporalLayer: -, consumerId: 5656cff8-8e5c-45a3-aa70-6383c4a0ddd0]
[06:58:53.464] Consumer Transport-WebRTC _trace_ event [transportId:4db2ee6b-4e67-4e39-8623-622230a4d011, msg.type:bwe, msg:{ direction: 'out', info: { availableBitrate: 1794460, desiredBitrate: 1796957, effectiveDesiredBitrate: 1796957, maxBitrate: 2425892, maxPaddingBitrate: 2062008, minBitrate: 30000, startBitrate: 1794460, type: 'transport-cc' }, timestamp: 5884044, type: 'bwe' }]
Any hint on how I could debug this better?
Thank you for your help!
