runfix: Allow blurred background to be enabled before the call starts (#17605)

Author: atomrc

src/script/calling/CallingRepository.ts

@@ -82,7 +82,6 @@ import {EventSource} from '../event/EventSource';
 import type {MediaDevicesHandler} from '../media/MediaDevicesHandler';
 import type {MediaStreamHandler} from '../media/MediaStreamHandler';
 import {MediaType} from '../media/MediaType';
-import {applyBlur} from '../media/VideoBackgroundBlur';
 import {APIClient} from '../service/APIClientSingleton';
 import {Core} from '../service/CoreSingleton';
 import {TeamState} from '../team/TeamState';
@@ -131,6 +130,7 @@ export class CallingRepository {
   private readonly acceptVersionWarning: (conversationId: QualifiedId) => void;
   private readonly callLog: string[];
   private readonly logger: Logger;
+  private enableBackgroundBlur = false;
   private avsVersion: number = 0;
   private incomingCallCallback: (call: Call) => void;
   private isReady: boolean = false;
@@ -276,13 +276,8 @@ export class CallingRepository {
     if (!videoFeed) {
       return;
     }
-    let newVideoFeed = videoFeed;
-    if (enable) {
-      const blurredVideoStream = await applyBlur(videoFeed);
-      // Keep a reference to the blurred stream in order to release it when the blur is disabled
-      selfParticipant.blurredVideoStream(blurredVideoStream);
-      newVideoFeed = blurredVideoStream.stream;
-    }
+    this.enableBackgroundBlur = enable;
+    const newVideoFeed = enable ? ((await selfParticipant.setBlurredBackground(true)) as MediaStream) : videoFeed;
     this.changeMediaSource(newVideoFeed, MediaType.VIDEO, false);
   }
 
@@ -530,10 +525,12 @@ export class CallingRepository {
   private async warmupMediaStreams(call: Call, audio: boolean, camera: boolean): Promise<boolean> {
     // if it's a video call we query the video user media in order to display the video preview
     try {
+      const selfParticipant = call.getSelfParticipant();
       camera = this.teamState.isVideoCallingEnabled() ? camera : false;
       const mediaStream = await this.getMediaStream({audio, camera}, call.isGroupOrConference);
       if (call.state() !== CALL_STATE.NONE) {
-        call.getSelfParticipant().updateMediaStream(mediaStream, true);
+        selfParticipant.updateMediaStream(mediaStream, true);
+        await selfParticipant.setBlurredBackground(this.enableBackgroundBlur);
         if (camera) {
           call.getSelfParticipant().videoState(VIDEO_STATE.STARTED);
         }
@@ -1780,8 +1777,9 @@ export class CallingRepository {
         }
         const mediaStream = await this.getMediaStream(missingStreams, call.isGroupOrConference);
         this.mediaStreamQuery = undefined;
-        const newStream = selfParticipant.updateMediaStream(mediaStream, true);
-        return newStream;
+        selfParticipant.updateMediaStream(mediaStream, true);
+        await selfParticipant.setBlurredBackground(this.enableBackgroundBlur);
+        return selfParticipant.getMediaStream();
       } catch (error) {
         this.mediaStreamQuery = undefined;
         this.logger.warn('Could not get mediaStream for call', error);

src/script/calling/Participant.ts

@@ -18,68 +18,72 @@
  */
 
 import {QualifiedId} from '@wireapp/api-client/lib/user';
-import ko from 'knockout';
+import ko, {observable, pureComputed} from 'knockout';
 
 import {VIDEO_STATE} from '@wireapp/avs';
 
 import {matchQualifiedIds} from 'Util/QualifiedId';
 
 import {User} from '../entity/User';
+import {applyBlur} from '../media/VideoBackgroundBlur';
 
 export type UserId = string;
 export type ClientId = string;
 
 export class Participant {
   // Video
-  public videoState: ko.Observable<VIDEO_STATE>;
-  public videoStream: ko.Observable<MediaStream | undefined>;
-  public blurredVideoStream = ko.observable<{stream: MediaStream; release: () => void} | undefined>(undefined);
-  public hasActiveVideo: ko.PureComputed<boolean>;
-  public hasPausedVideo: ko.PureComputed<boolean>;
-  public sharesScreen: ko.PureComputed<boolean>;
-  public sharesCamera: ko.PureComputed<boolean>;
-  public startedScreenSharingAt: ko.Observable<number>;
-  public isActivelySpeaking: ko.Observable<boolean>;
-  public isSendingVideo: ko.PureComputed<boolean>;
-  public isAudioEstablished: ko.Observable<boolean>;
+  public readonly videoState = observable(VIDEO_STATE.STOPPED);
+  public readonly videoStream = observable<MediaStream | undefined>();
+  public readonly blurredVideoStream = observable<{stream: MediaStream; release: () => void} | undefined>();
+  public readonly hasActiveVideo: ko.PureComputed<boolean>;
+  public readonly hasPausedVideo: ko.PureComputed<boolean>;
+  public readonly sharesScreen: ko.PureComputed<boolean>;
+  public readonly sharesCamera: ko.PureComputed<boolean>;
+  public readonly startedScreenSharingAt = observable<number>(0);
+  public readonly isActivelySpeaking = observable(false);
+  public readonly isSendingVideo: ko.PureComputed<boolean>;
+  public readonly isAudioEstablished = observable(false);
 
   // Audio
-  public audioStream: ko.Observable<MediaStream | undefined>;
-  public isMuted: ko.Observable<boolean>;
+  public readonly audioStream = observable<MediaStream | undefined>();
+  public readonly isMuted = observable(false);
 
   constructor(
-    public user: User,
-    public clientId: ClientId,
+    public readonly user: User,
+    public readonly clientId: ClientId,
   ) {
-    this.videoState = ko.observable(VIDEO_STATE.STOPPED);
-    this.hasActiveVideo = ko.pureComputed(() => {
+    this.hasActiveVideo = pureComputed(() => {
       return (this.sharesCamera() || this.sharesScreen()) && !!this.videoStream();
     });
-    this.sharesScreen = ko.pureComputed(() => {
+    this.sharesScreen = pureComputed(() => {
       return this.videoState() === VIDEO_STATE.SCREENSHARE;
     });
-    this.sharesCamera = ko.pureComputed(() => {
+    this.sharesCamera = pureComputed(() => {
       return [VIDEO_STATE.STARTED, VIDEO_STATE.PAUSED].includes(this.videoState());
     });
-    this.hasPausedVideo = ko.pureComputed(() => {
+    this.hasPausedVideo = pureComputed(() => {
       return this.videoState() === VIDEO_STATE.PAUSED;
     });
-    this.videoStream = ko.observable();
-    this.audioStream = ko.observable();
-    this.isActivelySpeaking = ko.observable(false);
-    this.startedScreenSharingAt = ko.observable();
-    this.isMuted = ko.observable(false);
-    this.isSendingVideo = ko.pureComputed(() => {
+    this.isSendingVideo = pureComputed(() => {
       return this.videoState() !== VIDEO_STATE.STOPPED;
     });
-    this.isAudioEstablished = ko.observable(false);
   }
 
   public releaseBlurredVideoStream(): void {
     this.blurredVideoStream()?.release();
     this.blurredVideoStream(undefined);
   }
 
+  public async setBlurredBackground(isBlurred: boolean) {
+    const originalVideoStream = this.videoStream();
+    if (isBlurred && originalVideoStream) {
+      this.blurredVideoStream(await applyBlur(originalVideoStream));
+    } else {
+      this.releaseBlurredVideoStream();
+    }
+    return this.blurredVideoStream()?.stream;
+  }
+
   readonly doesMatchIds = (userId: QualifiedId, clientId: ClientId): boolean =>
     matchQualifiedIds(userId, this.user.qualifiedId) && clientId === this.clientId;
 
@@ -89,6 +93,7 @@ export class Participant {
   }
 
   setVideoStream(videoStream: MediaStream, stopTracks: boolean): void {
+    this.releaseBlurredVideoStream();
     this.releaseStream(this.videoStream(), stopTracks);
     this.videoStream(videoStream);
   }
@@ -104,8 +109,9 @@ export class Participant {
   }
 
   getMediaStream(): MediaStream {
-    const audioTracks: MediaStreamTrack[] = this.audioStream() ? this.audioStream().getTracks() : [];
-    const videoTracks: MediaStreamTrack[] = this.videoStream() ? this.videoStream().getTracks() : [];
+    const audioTracks: MediaStreamTrack[] = this.audioStream()?.getTracks() ?? [];
+    const videoTracks: MediaStreamTrack[] =
+      this.blurredVideoStream()?.stream.getTracks() ?? this.videoStream()?.getTracks() ?? [];
     return new MediaStream(audioTracks.concat(videoTracks));
   }
 

src/script/media/BackgroundBlurrer/fragmentShader.glsl

@@ -1,4 +1,4 @@
-#define BLUR_QUALITY 8 //The higher the value, the more blur. Must be an even number.
+#define BLUR_QUALITY 14 //The higher the value, the more blur. Must be an even number.
 #define SMOOTH 3 // This will create a smooth transition between the blurred and the clear part (the higher the value, the smoother)
 
 precision mediump float;

src/script/media/VideoBackgroundBlur.ts

@@ -32,7 +32,7 @@ enum FRAMERATE {
 
 const QualitySettings = {
   segmentationModel: SEGMENTATION_MODEL.PERFORMANCE,
-  framerate: FRAMERATE.HIGH,
+  framerate: FRAMERATE.LOW,
 };
 
 // Calculate the FPS interval