feat: move closed caption handling to the SDK [wip]

packages/client/docusaurus/docs/javascript/02-guides/03-call-and-participant-state.mdx

@@ -51,6 +51,7 @@ Here is an excerpt of the call state properties:
 | `blockedUserIds$`        | `blockedUserIds`        | The list of blocked user IDs.                                                                                |
 | `callingState$`          | `callingState`          | Provides information about the call state. For example, `RINGING`, `JOINED` or `RECONNECTING`.               |
 | `callStatsReport$`       | `callStatsReport`       | When stats gathering is enabled, this observable will emit a new value at a regular (configurable) interval. |
+| `closedCaptions$`        | `closedCaptions`        | The closed captions state of the call.                                                                       |
 | `createdAt$`             | `createdAt`             | The time the call was created.                                                                               |
 | `createdBy$`             | `createdBy`             | The user who created the call.                                                                               |
 | `custom$`                | `custom`                | Custom data attached to the call.                                                                            |

packages/client/src/store/CallState.ts

@@ -19,6 +19,7 @@ import {
 import { CallStatsReport } from '../stats';
 import {
   BlockedUserEvent,
+  CallClosedCaption,
   CallHLSBroadcastingStartedEvent,
   CallIngressResponse,
   CallMemberAddedEvent,
@@ -32,6 +33,7 @@ import {
   CallSessionParticipantLeftEvent,
   CallSessionResponse,
   CallSettingsResponse,
+  ClosedCaptionEvent,
   EgressResponse,
   MemberResponse,
   OwnCapability,
@@ -68,6 +70,21 @@ type OrphanedTrack = {
   track: MediaStream;
 };
 
+type ClosedCaptionsConfig = {
+  /**
+   * The time in milliseconds to keep a closed caption in the queue.
+   * Default is 2700 ms.
+   */
+  retentionTime?: number;
+  /**
+   * The maximum number of closed captions to keep in the queue.
+   * When the queue is full, the oldest closed caption will be removed.
+   *
+   * Default is 2.
+   */
+  queueSize?: number;
+};
+
 /**
  * Holds the state of the current call.
  * @react You don't have to use this class directly, as we are exposing the state through Hooks.
@@ -117,6 +134,7 @@ export class CallState {
   private callStatsReportSubject = new BehaviorSubject<
     CallStatsReport | undefined
   >(undefined);
+  private closedCaptionsSubject = new BehaviorSubject<CallClosedCaption[]>([]);
 
   // These are tracks that were delivered to the Subscriber's onTrack event
   // that we couldn't associate with a participant yet.
@@ -285,6 +303,11 @@ export class CallState {
    */
   thumbnails$: Observable<ThumbnailResponse | undefined>;
 
+  /**
+   * The queue of closed captions.
+   */
+  closedCaptions$: Observable<CallClosedCaption[]>;
+
   readonly logger = getLogger(['CallState']);
 
   /**
@@ -294,6 +317,12 @@ export class CallState {
    */
   private sortParticipantsBy = defaultSortPreset;
 
+  /**
+   * The closed captions configuration.
+   * @private
+   */
+  private closedCaptionsConfig: ClosedCaptionsConfig = {};
+
   private readonly eventHandlers: {
     [EventType in WSEvent['type']]:
       | ((event: Extract<WSEvent, { type: EventType }>) => void)
@@ -357,6 +386,7 @@ export class CallState {
     this.settings$ = this.settingsSubject.asObservable();
     this.endedBy$ = this.endedBySubject.asObservable();
     this.thumbnails$ = this.thumbnailsSubject.asObservable();
+    this.closedCaptions$ = this.closedCaptionsSubject.asObservable();
 
     /**
      * Performs shallow comparison of two arrays.
@@ -393,7 +423,6 @@ export class CallState {
 
     this.eventHandlers = {
       // these events are not updating the call state:
-      'call.closed_caption': undefined,
       'call.deleted': undefined,
       'call.permission_request': undefined,
       'call.recording_ready': undefined,
@@ -415,6 +444,7 @@ export class CallState {
       // events that update call state:
       'call.accepted': (e) => this.updateFromCallResponse(e.call),
       'call.blocked_user': this.blockUser,
+      'call.closed_caption': this.updateClosedCaptions,
       'call.created': (e) => this.updateFromCallResponse(e.call),
       'call.ended': (e) => {
         this.updateFromCallResponse(e.call);
@@ -792,6 +822,13 @@ export class CallState {
     return this.getCurrentValue(this.thumbnails$);
   }
 
+  /**
+   * Returns the current queue of closed captions.
+   */
+  get closedCaptions() {
+    return this.getCurrentValue(this.closedCaptions$);
+  }
+
   /**
    * Will try to find the participant with the given sessionId in the current call.
    *
@@ -1040,6 +1077,15 @@ export class CallState {
     return orphans;
   };
 
+  /**
+   * Updates the closed captions configuration.
+   *
+   * @param config the new closed captions configuration.
+   */
+  updateClosedCaptionSettings = (config: Partial<ClosedCaptionsConfig>) => {
+    this.closedCaptionsConfig = { ...this.closedCaptionsConfig, ...config };
+  };
+
   /**
    * Updates the call state with the data received from the server.
    *
@@ -1299,4 +1345,25 @@ export class CallState {
       this.setCurrentValue(this.ownCapabilitiesSubject, event.own_capabilities);
     }
   };
+
+  private updateClosedCaptions = (event: ClosedCaptionEvent) => {
+    this.setCurrentValue(this.closedCaptionsSubject, (current) => {
+      const { closed_caption } = event;
+
+      const key = (c: CallClosedCaption) => `${c.speaker_id}/${c.start_time}`;
+      const newCcKey = key(closed_caption);
+      const isDuplicate = current.some((caption) => key(caption) === newCcKey);
+      if (isDuplicate) return current;
+
+      const { retentionTime = 2700, queueSize = 2 } = this.closedCaptionsConfig;
+      // TODO: we should probably cancel the timeout call is left
+      // schedule the removal of the closed caption after the retention time
+      setTimeout(() => {
+        this.setCurrentValue(this.closedCaptionsSubject, (captions) =>
+          captions.filter((caption) => caption !== closed_caption),
+        );
+      }, retentionTime);
+      return [...current, closed_caption].slice(-queueSize);
+    });
+  };
 }

packages/react-bindings/src/hooks/callStateHooks.ts

@@ -1,6 +1,7 @@
 import { useMemo } from 'react';
 import {
   Call,
+  CallClosedCaption,
   CallIngressResponse,
   CallSessionResponse,
   CallSettingsResponse,
@@ -479,3 +480,11 @@ export const useIncomingVideoSettings = () => {
   );
   return settings;
 };
+
+/**
+ * Returns the current call's closed captions queue.
+ */
+export const useCallClosedCaptions = (): CallClosedCaption[] => {
+  const { closedCaptions$ } = useCallState();
+  return useObservableValue(closedCaptions$);
+};

packages/react-native-sdk/docusaurus/docs/reactnative/03-core/03-call-and-participant-state.mdx

@@ -66,6 +66,7 @@ Here is an excerpt of the available call state hooks:
 | `useCall`                            | The `Call` instance that is registered with `StreamCall`. You need the `Call` instance to initiate API calls. |
 | `useCallBlockedUserIds`              | The list of blocked user IDs.                                                                                 |
 | `useCallCallingState`                | Provides information about the call state. For example, `RINGING`, `JOINED` or `RECONNECTING`.                |
+| `useCallClosedCaptions`              | The closed captions of the call.                                                                              |
 | `useCallCreatedAt`                   | The time the call was created.                                                                                |
 | `useCallCreatedBy`                   | The user that created the call.                                                                               |
 | `useCallCustomData`                  | The custom data attached to the call.                                                                         |
@@ -205,7 +206,7 @@ const hosts = participants.filter((p) => p.roles.includes('host'));
 
 // participants that publish video and audio
 const videoParticipants = participants.filter(
-  (p) => hasVideo(p) && hasAudio(p),
+  (p) => hasVideo(p) && hasAudio(p)
 );
 ```
 

packages/react-sdk/docusaurus/docs/React/02-guides/03-call-and-participant-state.mdx

@@ -67,6 +67,7 @@ Here is an excerpt of the available call state hooks:
 | `useCall`                            | The `Call` instance that is registered with `StreamCall`. You need the `Call` instance to initiate API calls. |
 | `useCallBlockedUserIds`              | The list of blocked user IDs.                                                                                 |
 | `useCallCallingState`                | Provides information about the call state. For example, `RINGING`, `JOINED` or `RECONNECTING`.                |
+| `useCallClosedCaptions`              | The closed captions of the call.                                                                              |
 | `useCallCreatedAt`                   | The time the call was created.                                                                                |
 | `useCallCreatedBy`                   | The user that created the call.                                                                               |
 | `useCallCustomData`                  | The custom data attached to the call.                                                                         |

sample-apps/react/react-dogfood/components/ClosedCaptions.tsx

@@ -27,40 +27,21 @@ const useDeduplicatedQueue = (initialQueue: CallClosedCaption[] = []) => {
 };
 
 export const ClosedCaptions = () => {
-  const call = useCall();
-  const [queue, addToQueue, setQueue] = useDeduplicatedQueue();
-
-  useEffect(() => {
-    if (!call) return;
-    return call.on('call.closed_caption', (e) => {
-      if (e.type !== 'call.closed_caption') return;
-      if (e.closed_caption.text.trim() === '') return;
-      addToQueue(e.closed_caption);
-    });
-  }, [call, addToQueue]);
-
-  useEffect(() => {
-    const id = setTimeout(() => {
-      setQueue((prevQueue) =>
-        prevQueue.length !== 0 ? prevQueue.slice(1) : prevQueue,
-      );
-    }, 2700);
-    return () => clearTimeout(id);
-  }, [queue, setQueue]);
-
+  const { useCallClosedCaptions } = useCallStateHooks();
+  const closedCaptions = useCallClosedCaptions();
   const userNameMapping = useUserIdToUserNameMapping();
 
   return (
     <div className="rd__closed-captions">
-      {queue.slice(-2).map(({ speaker_id, text, start_time }) => (
+      {closedCaptions.map(({ speaker_id, text, start_time }) => (
         <p
           className="rd__closed-captions__line"
           key={`${speaker_id}-${start_time}`}
         >
           <span className="rd__closed-captions__speaker">
             {userNameMapping[speaker_id] || speaker_id}:
           </span>
-          <span className="rd__closed-captions__text"> {text}</span>
+          <span className="rd__closed-captions__text">{text}</span>
         </p>
       ))}
     </div>
@@ -74,8 +55,6 @@ export const ClosedCaptionsSidebar = () => {
   useEffect(() => {
     if (!call) return;
     return call.on('call.closed_caption', (e) => {
-      if (e.type !== 'call.closed_caption') return;
-      if (e.closed_caption.text.trim() === '') return;
       addToQueue(e.closed_caption);
     });
   }, [call, addToQueue]);