Skip to content

Commit 10d5cb8

Browse files
oliverlazoliverlaz
committed
feat: enrich closed captions with speaker name
1 parent 64eb786 commit 10d5cb8

File tree

9 files changed

+181
-111
lines changed

9 files changed

+181
-111
lines changed

‎packages/client/docusaurus/docs/javascript/02-guides/03-call-and-participant-state.mdx

Lines changed: 28 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -45,33 +45,34 @@ Otherwise, `call.state` observables will emit empty values and you won't get rea
4545

4646
Here is an excerpt of the call state properties:
4747

48-
| Reactive value | Static value | Description |
49-
| ------------------------ | ----------------------- | ------------------------------------------------------------------------------------------------------------ |
50-
| `backstage$` | `backstage` | `true` when the call runs in `backstage` mode |
51-
| `blockedUserIds$` | `blockedUserIds` | The list of blocked user IDs. |
52-
| `callingState$` | `callingState` | Provides information about the call state. For example, `RINGING`, `JOINED` or `RECONNECTING`. |
53-
| `callStatsReport$` | `callStatsReport` | When stats gathering is enabled, this observable will emit a new value at a regular (configurable) interval. |
54-
| `closedCaptions$` | `closedCaptions` | The closed captions state of the call. |
55-
| `createdAt$` | `createdAt` | The time the call was created. |
56-
| `createdBy$` | `createdBy` | The user who created the call. |
57-
| `custom$` | `custom` | Custom data attached to the call. |
58-
| `dominantSpeaker$` | `dominantSpeaker` | The participant that is the current dominant speaker of the call. |
59-
| `egress$` | `egress` | The egress data of the call (for broadcasting and livestreaming). |
60-
| `endedAt$` | `endedAt` | The time the call was ended. |
61-
| `endedBy$` | `endedBy` | The user who ended the call. |
62-
| `hasOngoingScreenShare$` | `hasOngoingScreenShare` | It will return `true` if at least one participant is sharing their screen. |
63-
| `ingress$` | `ingress` | The ingress data of the call (for broadcasting and livestreaming). |
64-
| `members$` | `members` | The list of call members |
65-
| `ownCapabilities$` | `ownCapabilities` | The capabilities of the local participant. |
66-
| `pinnedParticipants$` | `pinnedParticipants` | The participants that are currently pinned. |
67-
| `recording$` | `recording` | The recording state of the call. |
68-
| `session$` | `session` | The data for the current call session. |
69-
| `settings$` | `settings` | The settings of the call. |
70-
| `startedAt$` | `startedAt` | The actual start time of the current call session. |
71-
| `startsAt$` | `startsAt` | The time the call is scheduled to start. |
72-
| `thumbnails$` | `thumbnails` | The thumbnails of the call. |
73-
| `transcribing$` | `transcribing` | The transcribing state of the call. |
74-
| `updatedAt$` | `updatedAt` | The time the call was updated. |
48+
| Reactive value | Static value | Description |
49+
| ------------------------------ | ----------------------------- | ------------------------------------------------------------------------------------------------------------ |
50+
| `backstage$` | `backstage` | `true` when the call runs in `backstage` mode |
51+
| `blockedUserIds$` | `blockedUserIds` | The list of blocked user IDs. |
52+
| `callingState$` | `callingState` | Provides information about the call state. For example, `RINGING`, `JOINED` or `RECONNECTING`. |
53+
| `callStatsReport$` | `callStatsReport` | When stats gathering is enabled, this observable will emit a new value at a regular (configurable) interval. |
54+
| `closedCaptions$` | `closedCaptions` | The closed captions state of the call. |
55+
| `createdAt$` | `createdAt` | The time the call was created. |
56+
| `createdBy$` | `createdBy` | The user who created the call. |
57+
| `custom$` | `custom` | Custom data attached to the call. |
58+
| `dominantSpeaker$` | `dominantSpeaker` | The participant that is the current dominant speaker of the call. |
59+
| `egress$` | `egress` | The egress data of the call (for broadcasting and livestreaming). |
60+
| `endedAt$` | `endedAt` | The time the call was ended. |
61+
| `endedBy$` | `endedBy` | The user who ended the call. |
62+
| `hasOngoingScreenShare$` | `hasOngoingScreenShare` | It will return `true` if at least one participant is sharing their screen. |
63+
| `ingress$` | `ingress` | The ingress data of the call (for broadcasting and livestreaming). |
64+
| `members$` | `members` | The list of call members |
65+
| `ownCapabilities$` | `ownCapabilities` | The capabilities of the local participant. |
66+
| `pinnedParticipants$` | `pinnedParticipants` | The participants that are currently pinned. |
67+
| `recording$` | `recording` | The recording state of the call. |
68+
| `session$` | `session` | The data for the current call session. |
69+
| `sessionParticipantsByUserId$` | `sessionParticipantsByUserId` | The participants of the call session by user ID. |
70+
| `settings$` | `settings` | The settings of the call. |
71+
| `startedAt$` | `startedAt` | The actual start time of the current call session. |
72+
| `startsAt$` | `startsAt` | The time the call is scheduled to start. |
73+
| `thumbnails$` | `thumbnails` | The thumbnails of the call. |
74+
| `transcribing$` | `transcribing` | The transcribing state of the call. |
75+
| `updatedAt$` | `updatedAt` | The time the call was updated. |
7576

7677
:::note
7778
Your IDE of choice may help you to discover the other properties of the call state.

‎packages/client/src/Call.ts

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import {
1515
import {
1616
CallingState,
1717
CallState,
18-
ClosedCaptionsSettings,
1918
StreamVideoWriteableStateStore,
2019
} from './store';
2120
import {
@@ -84,6 +83,7 @@ import {
8483
AudioTrackType,
8584
CallConstructor,
8685
CallLeaveOptions,
86+
ClosedCaptionsSettings,
8787
JoinCallData,
8888
PublishOptions,
8989
TrackMuteType,
@@ -1533,15 +1533,6 @@ export class Call {
15331533
return this.state.setSortParticipantsBy(criteria);
15341534
};
15351535

1536-
/**
1537-
* Updates the closed caption settings.
1538-
*
1539-
* @param config the closed caption settings to apply
1540-
*/
1541-
updateClosedCaptionSettings = (config: Partial<ClosedCaptionsSettings>) => {
1542-
this.state.updateClosedCaptionSettings(config);
1543-
};
1544-
15451536
/**
15461537
* Updates the list of video layers to publish.
15471538
*
@@ -1718,6 +1709,15 @@ export class Call {
17181709
);
17191710
};
17201711

1712+
/**
1713+
* Updates the closed caption settings.
1714+
*
1715+
* @param config the closed caption settings to apply
1716+
*/
1717+
updateClosedCaptionSettings = (config: Partial<ClosedCaptionsSettings>) => {
1718+
this.state.updateClosedCaptionSettings(config);
1719+
};
1720+
17211721
/**
17221722
* Sends a `call.permission_request` event to all users connected to the call.
17231723
* The call settings object contains information about which permissions can be requested during a call

‎packages/client/src/store/CallState.ts

Lines changed: 54 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ import type { Patch } from './rxUtils';
99
import * as RxUtils from './rxUtils';
1010
import { CallingState } from './CallingState';
1111
import {
12+
type ClosedCaptionsSettings,
13+
type StreamCallClosedCaption,
1214
type StreamVideoParticipant,
1315
type StreamVideoParticipantPatch,
1416
type StreamVideoParticipantPatches,
@@ -26,6 +28,7 @@ import {
2628
CallMemberRemovedEvent,
2729
CallMemberUpdatedEvent,
2830
CallMemberUpdatedPermissionEvent,
31+
CallParticipantResponse,
2932
CallReactionEvent,
3033
CallResponse,
3134
CallSessionParticipantCountsUpdatedEvent,
@@ -70,21 +73,6 @@ type OrphanedTrack = {
7073
track: MediaStream;
7174
};
7275

73-
export type ClosedCaptionsSettings = {
74-
/**
75-
* The time in milliseconds to keep a closed caption in the queue.
76-
* Default is 2700 ms.
77-
*/
78-
retentionTimeInMs?: number;
79-
/**
80-
* The maximum number of closed captions to keep in the queue.
81-
* When the queue is full, the oldest closed caption will be removed.
82-
*
83-
* Default is 2.
84-
*/
85-
queueSize?: number;
86-
};
87-
8876
/**
8977
* Holds the state of the current call.
9078
* @react You don't have to use this class directly, as we are exposing the state through Hooks.
@@ -134,7 +122,9 @@ export class CallState {
134122
private callStatsReportSubject = new BehaviorSubject<
135123
CallStatsReport | undefined
136124
>(undefined);
137-
private closedCaptionsSubject = new BehaviorSubject<CallClosedCaption[]>([]);
125+
private closedCaptionsSubject = new BehaviorSubject<
126+
StreamCallClosedCaption[]
127+
>([]);
138128

139129
// These are tracks that were delivered to the Subscriber's onTrack event
140130
// that we couldn't associate with a participant yet.
@@ -283,6 +273,13 @@ export class CallState {
283273
*/
284274
session$: Observable<CallSessionResponse | undefined>;
285275

276+
/**
277+
* Returns the session participants indexed by their user ID.
278+
*/
279+
sessionParticipantsByUserId$: Observable<
280+
Record<string, CallParticipantResponse>
281+
>;
282+
286283
/**
287284
* Will provide the settings of this call.
288285
*/
@@ -306,7 +303,7 @@ export class CallState {
306303
/**
307304
* The queue of closed captions.
308305
*/
309-
closedCaptions$: Observable<CallClosedCaption[]>;
306+
closedCaptions$: Observable<StreamCallClosedCaption[]>;
310307

311308
readonly logger = getLogger(['CallState']);
312309

@@ -318,8 +315,8 @@ export class CallState {
318315
/**
319316
* The closed captions configuration.
320317
*/
321-
private closedCaptionsSettings: ClosedCaptionsSettings = {};
322-
private closedCaptionsCleanupTasks = new Map<string, NodeJS.Timeout>();
318+
private closedCaptionsSettings: ClosedCaptionsSettings | undefined;
319+
private closedCaptionsTasks = new Map<string, NodeJS.Timeout>();
323320

324321
private readonly eventHandlers: {
325322
[EventType in WSEvent['type']]:
@@ -386,6 +383,20 @@ export class CallState {
386383
this.thumbnails$ = this.thumbnailsSubject.asObservable();
387384
this.closedCaptions$ = this.closedCaptionsSubject.asObservable();
388385

386+
// look-up cache for session participants
387+
this.sessionParticipantsByUserId$ = this.session$.pipe(
388+
map((session) => {
389+
if (!session) return {};
390+
return session.participants.reduce<
391+
Record<string, CallParticipantResponse>
392+
>((target, participant) => {
393+
target[participant.user.id] = participant;
394+
return target;
395+
}, {});
396+
}),
397+
shareReplay({ bufferSize: 1, refCount: true }),
398+
);
399+
389400
/**
390401
* Performs shallow comparison of two arrays.
391402
* Expects primitive values: [1, 2, 3] is equal to [2, 1, 3].
@@ -496,9 +507,9 @@ export class CallState {
496507
* Runs the cleanup tasks.
497508
*/
498509
dispose = () => {
499-
for (const [ccKey, taskId] of this.closedCaptionsCleanupTasks.entries()) {
510+
for (const [ccKey, taskId] of this.closedCaptionsTasks.entries()) {
500511
clearTimeout(taskId);
501-
this.closedCaptionsCleanupTasks.delete(ccKey);
512+
this.closedCaptionsTasks.delete(ccKey);
502513
}
503514
};
504515

@@ -802,6 +813,13 @@ export class CallState {
802813
return this.getCurrentValue(this.session$);
803814
}
804815

816+
/**
817+
* Will provide the session participants of this call indexed by their user ID.
818+
*/
819+
get sessionParticipantsByUserId() {
820+
return this.getCurrentValue(this.sessionParticipantsByUserId$);
821+
}
822+
805823
/**
806824
* Will provide the settings of this call.
807825
*/
@@ -1363,26 +1381,33 @@ export class CallState {
13631381
if (duplicate) return queue;
13641382

13651383
const { retentionTimeInMs = 2700, queueSize = 2 } =
1366-
this.closedCaptionsSettings;
1367-
1368-
const nextQueue = [...queue, closed_caption];
1384+
this.closedCaptionsSettings || {};
1385+
1386+
const participant =
1387+
this.sessionParticipantsByUserId[closed_caption.speaker_id];
1388+
const speaker_name = participant?.user.name || closed_caption.speaker_id;
1389+
const nextClosedCaption: StreamCallClosedCaption = {
1390+
...event.closed_caption,
1391+
speaker_name,
1392+
};
1393+
const nextQueue = [...queue, nextClosedCaption];
13691394

13701395
// schedule the removal of the closed caption after the retention time
13711396
if (retentionTimeInMs > 0) {
13721397
const taskId = setTimeout(() => {
13731398
this.setCurrentValue(this.closedCaptionsSubject, (captions) =>
1374-
captions.filter((caption) => caption !== closed_caption),
1399+
captions.filter((caption) => caption !== nextClosedCaption),
13751400
);
1376-
this.closedCaptionsCleanupTasks.delete(currentKey);
1401+
this.closedCaptionsTasks.delete(currentKey);
13771402
}, retentionTimeInMs);
1378-
this.closedCaptionsCleanupTasks.set(currentKey, taskId);
1403+
this.closedCaptionsTasks.set(currentKey, taskId);
13791404

13801405
// cancel the cleanup tasks for the closed captions that are no longer in the queue
13811406
for (let i = 0; i < nextQueue.length - queueSize; i++) {
13821407
const key = keyOf(nextQueue[i]);
1383-
const task = this.closedCaptionsCleanupTasks.get(key);
1408+
const task = this.closedCaptionsTasks.get(key);
13841409
clearTimeout(task);
1385-
this.closedCaptionsCleanupTasks.delete(key);
1410+
this.closedCaptionsTasks.delete(key);
13861411
}
13871412
}
13881413

‎packages/client/src/store/__tests__/CallState.test.ts

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1000,7 +1000,7 @@ describe('CallState', () => {
10001000
});
10011001
}
10021002
expect(state.closedCaptions.length).toBe(2);
1003-
expect(state['closedCaptionsCleanupTasks'].size).toBe(2);
1003+
expect(state['closedCaptionsTasks'].size).toBe(2);
10041004
expect(state.closedCaptions.map((cc) => cc.text)).toEqual([
10051005
'Hello world 3',
10061006
'Hello world 4',
@@ -1021,11 +1021,11 @@ describe('CallState', () => {
10211021
},
10221022
});
10231023
expect(state.closedCaptions.length).toBe(1);
1024-
expect(state['closedCaptionsCleanupTasks'].size).toBe(1);
1024+
expect(state['closedCaptionsTasks'].size).toBe(1);
10251025

10261026
vi.runAllTimers();
10271027
expect(state.closedCaptions.length).toBe(0);
1028-
expect(state['closedCaptionsCleanupTasks'].size).toBe(0);
1028+
expect(state['closedCaptionsTasks'].size).toBe(0);
10291029
});
10301030

10311031
it('should remove stale captions from the queue after timer runs', () => {
@@ -1043,11 +1043,40 @@ describe('CallState', () => {
10431043
},
10441044
});
10451045
expect(state.closedCaptions.length).toBe(1);
1046-
expect(state['closedCaptionsCleanupTasks'].size).toBe(1);
1046+
expect(state['closedCaptionsTasks'].size).toBe(1);
10471047

10481048
vi.advanceTimersByTime(101);
10491049
expect(state.closedCaptions.length).toBe(0);
1050-
expect(state['closedCaptionsCleanupTasks'].size).toBe(0);
1050+
expect(state['closedCaptionsTasks'].size).toBe(0);
1051+
});
1052+
1053+
it('should enrich closed captions with speaker name', () => {
1054+
const state = new CallState();
1055+
state.updateFromEvent({
1056+
type: 'call.session_started',
1057+
call: {
1058+
session: {
1059+
// @ts-expect-error incomplete data
1060+
participants: [{ user: { id: '123', name: 'Alice' } }],
1061+
participants_count_by_role: { user: 1 },
1062+
},
1063+
},
1064+
});
1065+
1066+
// @ts-expect-error incomplete data
1067+
state.updateFromEvent({
1068+
type: 'call.closed_caption',
1069+
closed_caption: {
1070+
speaker_id: `123`,
1071+
text: `Hello world`,
1072+
start_time: '2021-01-01T00:00:00.000Z',
1073+
end_time: '2021-01-01T00:02:00.000Z',
1074+
},
1075+
});
1076+
1077+
const closedCaptions = state.closedCaptions;
1078+
expect(closedCaptions.length).toBe(1);
1079+
expect(closedCaptions[0].speaker_name).toBe('Alice');
10511080
});
10521081

10531082
it('dispose cancels all cleanup tasks', () => {
@@ -1063,10 +1092,10 @@ describe('CallState', () => {
10631092
},
10641093
});
10651094
expect(state.closedCaptions.length).toBe(1);
1066-
expect(state['closedCaptionsCleanupTasks'].size).toBe(1);
1095+
expect(state['closedCaptionsTasks'].size).toBe(1);
10671096

10681097
state.dispose();
1069-
expect(state['closedCaptionsCleanupTasks'].size).toBe(0);
1098+
expect(state['closedCaptionsTasks'].size).toBe(0);
10701099
});
10711100
});
10721101
});

0 commit comments

Comments
 (0)