Skip to content

Commit 53aa34f

Browse files
SimonBrandnerSimonBrandner
authored
Support for mid-call devices changes (#2154)
* Push to `usermediaSenders` in `upgradeCall()` Signed-off-by: Šimon Brandner <[email protected]> * Make sure to enable tracks after a call upgrade Signed-off-by: Šimon Brandner <[email protected]> * Simplify `updateMuteStatus()` Signed-off-by: Šimon Brandner <[email protected]> * Add copyright for 2022 Signed-off-by: Šimon Brandner <[email protected]> * Add `updateLocalUsermediaStream()` Signed-off-by: Šimon Brandner <[email protected]> * Support mid-call device changes Signed-off-by: Šimon Brandner <[email protected]> * Use `updateLocalUsermediaStream()` for call upgrades Signed-off-by: Šimon Brandner <[email protected]> * Improve mock classes Signed-off-by: Šimon Brandner <[email protected]> * Add new tests Signed-off-by: Šimon Brandner <[email protected]>
1 parent 58756a1 commit 53aa34f

File tree

4 files changed

+268
-47
lines changed

4 files changed

+268
-47
lines changed

‎spec/unit/webrtc/call.spec.ts

Lines changed: 114 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -82,17 +82,34 @@ class MockRTCPeerConnection {
8282
}
8383
close() {}
8484
getStats() { return []; }
85+
addTrack(track: MockMediaStreamTrack) {return new MockRTCRtpSender(track);}
86+
}
87+
88+
class MockRTCRtpSender {
89+
constructor(public track: MockMediaStreamTrack) {}
90+
91+
replaceTrack(track: MockMediaStreamTrack) {this.track = track;}
92+
}
93+
94+
class MockMediaStreamTrack {
95+
constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) {}
96+
97+
stop() {}
8598
}
8699

87100
class MockMediaStream {
88101
constructor(
89102
public id: string,
103+
private tracks: MockMediaStreamTrack[] = [],
90104
) {}
91105

92-
getTracks() { return []; }
93-
getAudioTracks() { return [{ enabled: true }]; }
94-
getVideoTracks() { return [{ enabled: true }]; }
106+
getTracks() { return this.tracks; }
107+
getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); }
108+
getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); }
95109
addEventListener() {}
110+
removeEventListener() { }
111+
addTrack(track: MockMediaStreamTrack) {this.tracks.push(track);}
112+
removeTrack(track: MockMediaStreamTrack) {this.tracks.splice(this.tracks.indexOf(track), 1);}
96113
}
97114

98115
class MockMediaDeviceInfo {
@@ -102,7 +119,13 @@ class MockMediaDeviceInfo {
102119
}
103120

104121
class MockMediaHandler {
105-
getUserMediaStream() { return new MockMediaStream("mock_stream_from_media_handler"); }
122+
getUserMediaStream(audio: boolean, video: boolean) {
123+
const tracks = [];
124+
if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio"));
125+
if (video) tracks.push(new MockMediaStreamTrack("video_track", "video"));
126+
127+
return new MockMediaStream("mock_stream_from_media_handler", tracks);
128+
}
106129
stopUserMediaStream() {}
107130
}
108131

@@ -350,7 +373,15 @@ describe('Call', function() {
350373
},
351374
});
352375

353-
call.pushRemoteFeed(new MockMediaStream("remote_stream"));
376+
call.pushRemoteFeed(
377+
new MockMediaStream(
378+
"remote_stream",
379+
[
380+
new MockMediaStreamTrack("remote_audio_track", "audio"),
381+
new MockMediaStreamTrack("remote_video_track", "video"),
382+
],
383+
),
384+
);
354385
const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream");
355386
expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Usermedia);
356387
expect(feed?.isAudioMuted()).toBeTruthy();
@@ -396,4 +427,82 @@ describe('Call', function() {
396427
expect(client.client.mediaHandler.getUserMediaStream).toHaveBeenNthCalledWith(1, true, true);
397428
expect(client.client.mediaHandler.getUserMediaStream).toHaveBeenNthCalledWith(2, true, false);
398429
});
430+
431+
it("should handle mid-call device changes", async () => {
432+
client.client.mediaHandler.getUserMediaStream = jest.fn().mockReturnValue(
433+
new MockMediaStream(
434+
"stream", [
435+
new MockMediaStreamTrack("audio_track", "audio"),
436+
new MockMediaStreamTrack("video_track", "video"),
437+
],
438+
),
439+
);
440+
441+
const callPromise = call.placeVideoCall();
442+
await client.httpBackend.flush();
443+
await callPromise;
444+
445+
await call.onAnswerReceived({
446+
getContent: () => {
447+
return {
448+
version: 1,
449+
call_id: call.callId,
450+
party_id: 'party_id',
451+
answer: {
452+
sdp: DUMMY_SDP,
453+
},
454+
};
455+
},
456+
});
457+
458+
await call.updateLocalUsermediaStream(
459+
new MockMediaStream(
460+
"replacement_stream",
461+
[
462+
new MockMediaStreamTrack("new_audio_track", "audio"),
463+
new MockMediaStreamTrack("video_track", "video"),
464+
],
465+
),
466+
);
467+
expect(call.localUsermediaStream.id).toBe("stream");
468+
expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("new_audio_track");
469+
expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track");
470+
expect(call.usermediaSenders.find((sender) => {
471+
return sender?.track?.kind === "audio";
472+
}).track.id).toBe("new_audio_track");
473+
expect(call.usermediaSenders.find((sender) => {
474+
return sender?.track?.kind === "video";
475+
}).track.id).toBe("video_track");
476+
});
477+
478+
it("should handle upgrade to video call", async () => {
479+
const callPromise = call.placeVoiceCall();
480+
await client.httpBackend.flush();
481+
await callPromise;
482+
483+
await call.onAnswerReceived({
484+
getContent: () => {
485+
return {
486+
version: 1,
487+
call_id: call.callId,
488+
party_id: 'party_id',
489+
answer: {
490+
sdp: DUMMY_SDP,
491+
},
492+
[SDPStreamMetadataKey]: {},
493+
};
494+
},
495+
});
496+
497+
await call.upgradeCall(false, true);
498+
499+
expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("audio_track");
500+
expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track");
501+
expect(call.usermediaSenders.find((sender) => {
502+
return sender?.track?.kind === "audio";
503+
}).track.id).toBe("audio_track");
504+
expect(call.usermediaSenders.find((sender) => {
505+
return sender?.track?.kind === "video";
506+
}).track.id).toBe("video_track");
507+
});
399508
});

‎src/client.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -934,7 +934,7 @@ export class MatrixClient extends TypedEventEmitter<EmittedEvents, ClientEventHa
934934
protected checkTurnServersIntervalID: number;
935935
protected exportedOlmDeviceToImport: IOlmDevice;
936936
protected txnCtr = 0;
937-
protected mediaHandler = new MediaHandler();
937+
protected mediaHandler = new MediaHandler(this);
938938
protected pendingEventEncryption = new Map<string, Promise<void>>();
939939

940940
constructor(opts: IMatrixClientCreateOpts) {

‎src/webrtc/call.ts

Lines changed: 67 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
Copyright 2015, 2016 OpenMarket Ltd
33
Copyright 2017 New Vector Ltd
44
Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
5+
Copyright 2021 - 2022 Šimon Brandner <[email protected]>
56
67
Licensed under the Apache License, Version 2.0 (the "License");
78
you may not use this file except in compliance with the License.
@@ -950,29 +951,13 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
950951
if (!this.opponentSupportsSDPStreamMetadata()) return;
951952

952953
try {
953-
const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack;
954-
const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack;
955-
logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`);
956-
957-
const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo, false);
958-
if (upgradeAudio && upgradeVideo) {
959-
if (this.hasLocalUserMediaAudioTrack) return;
960-
if (this.hasLocalUserMediaVideoTrack) return;
961-
962-
this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
963-
} else if (upgradeAudio) {
964-
if (this.hasLocalUserMediaAudioTrack) return;
965-
966-
const audioTrack = stream.getAudioTracks()[0];
967-
this.localUsermediaStream.addTrack(audioTrack);
968-
this.peerConn.addTrack(audioTrack, this.localUsermediaStream);
969-
} else if (upgradeVideo) {
970-
if (this.hasLocalUserMediaVideoTrack) return;
971-
972-
const videoTrack = stream.getVideoTracks()[0];
973-
this.localUsermediaStream.addTrack(videoTrack);
974-
this.peerConn.addTrack(videoTrack, this.localUsermediaStream);
975-
}
954+
const getAudio = audio || this.hasLocalUserMediaAudioTrack;
955+
const getVideo = video || this.hasLocalUserMediaVideoTrack;
956+
957+
// updateLocalUsermediaStream() will take the tracks, use them as
958+
// replacement and throw the stream away, so it isn't reusable
959+
const stream = await this.client.getMediaHandler().getUserMediaStream(getAudio, getVideo, false);
960+
await this.updateLocalUsermediaStream(stream, audio, video);
976961
} catch (error) {
977962
logger.error("Failed to upgrade the call", error);
978963
this.emit(CallEvent.Error,
@@ -1088,6 +1073,63 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
10881073
}
10891074
}
10901075

1076+
/**
1077+
* Replaces/adds the tracks from the passed stream to the localUsermediaStream
1078+
* @param {MediaStream} stream to use a replacement for the local usermedia stream
1079+
*/
1080+
public async updateLocalUsermediaStream(
1081+
stream: MediaStream, forceAudio = false, forceVideo = false,
1082+
): Promise<void> {
1083+
const callFeed = this.localUsermediaFeed;
1084+
const audioEnabled = forceAudio || (!callFeed.isAudioMuted() && !this.remoteOnHold);
1085+
const videoEnabled = forceVideo || (!callFeed.isVideoMuted() && !this.remoteOnHold);
1086+
setTracksEnabled(stream.getAudioTracks(), audioEnabled);
1087+
setTracksEnabled(stream.getVideoTracks(), videoEnabled);
1088+
1089+
// We want to keep the same stream id, so we replace the tracks rather than the whole stream
1090+
for (const track of this.localUsermediaStream.getTracks()) {
1091+
this.localUsermediaStream.removeTrack(track);
1092+
track.stop();
1093+
}
1094+
for (const track of stream.getTracks()) {
1095+
this.localUsermediaStream.addTrack(track);
1096+
}
1097+
1098+
const newSenders = [];
1099+
1100+
for (const track of stream.getTracks()) {
1101+
const oldSender = this.usermediaSenders.find((sender) => sender.track?.kind === track.kind);
1102+
let newSender: RTCRtpSender;
1103+
1104+
if (oldSender) {
1105+
logger.info(
1106+
`Replacing track (` +
1107+
`id="${track.id}", ` +
1108+
`kind="${track.kind}", ` +
1109+
`streamId="${stream.id}", ` +
1110+
`streamPurpose="${callFeed.purpose}"` +
1111+
`) to peer connection`,
1112+
);
1113+
await oldSender.replaceTrack(track);
1114+
newSender = oldSender;
1115+
} else {
1116+
logger.info(
1117+
`Adding track (` +
1118+
`id="${track.id}", ` +
1119+
`kind="${track.kind}", ` +
1120+
`streamId="${stream.id}", ` +
1121+
`streamPurpose="${callFeed.purpose}"` +
1122+
`) to peer connection`,
1123+
);
1124+
newSender = this.peerConn.addTrack(track, this.localUsermediaStream);
1125+
}
1126+
1127+
newSenders.push(newSender);
1128+
}
1129+
1130+
this.usermediaSenders = newSenders;
1131+
}
1132+
10911133
/**
10921134
* Set whether our outbound video should be muted or not.
10931135
* @param {boolean} muted True to mute the outbound video.
@@ -1216,8 +1258,8 @@ export class MatrixCall extends TypedEventEmitter<CallEvent, CallEventHandlerMap
12161258
[SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(),
12171259
});
12181260

1219-
const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold;
1220-
const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold;
1261+
const micShouldBeMuted = this.isMicrophoneMuted() || this.remoteOnHold;
1262+
const vidShouldBeMuted = this.isLocalVideoMuted() || this.remoteOnHold;
12211263

12221264
setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted);
12231265
setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted);

0 commit comments

Comments
 (0)