Update publish-alpha (#2912)

* feat: custom ice servers in join config * fix: pass ice server config * fix: active state for poll, chat, peer list CTAs (#2872) * fix: add autofocus to participant search * fix: autofocus flag in participant_list * fix: autofocus on tab switch * fix: clean up * fix: active state for polls, chat and peerlist * feat: resizable input (#2911) * feat: resizable input * fix: handle resize on reopening chat * fix: use usecallback * fix: leaderboard scroll for mweb * fix: avg_jitter_buffer_delay in ms * feat: start/stop events for transcriptions in sdk * fix: show other options(emoji, handraise) when chat is paused or peer is blocked * feat: empty screen for viewers * fix: extra get poll responses calls * fix: handle video/audio getting paused on incoming call interruption --------- Co-authored-by: Kaustubh Kumar <[email protected]> Co-authored-by: Eswar Prasad Clinton. A <[email protected]> Co-authored-by: Amar Bathwal <[email protected]> Co-authored-by: Ravi theja <[email protected]>
100mslive · May 21, 2024 · dd51a87 · dd51a87
1 parent dc34271
commit dd51a87
Show file tree

Hide file tree

Showing 48 changed files with 607 additions and 343 deletions.
diff --git a/packages/hms-video-store/src/IHMSActions.ts b/packages/hms-video-store/src/IHMSActions.ts
@@ -1,3 +1,4 @@
+import { TranscriptionConfig } from './interfaces/transcription-config';
 import {
   HLSConfig,
   HLSTimedMetadata,
@@ -371,6 +372,18 @@ export interface IHMSActions<T extends HMSGenericTypes = { sessionStore: Record<
    */
   stopHLSStreaming(params?: HLSConfig): Promise<void>;
 
+  /**
+   * If you want to start transcriptions(Closed Caption).
+   * @param params.mode This is the mode which represent the type of transcription. Currently we have Caption mode only
+   */
+  startTranscription(params: TranscriptionConfig): Promise<void>;
+
+  /**
+   * If you want to stop transcriptions(Closed Caption).
+   * @param params.mode This is the mode which represent the type of transcription you want to stop. Currently we have Caption mode only
+   */
+  stopTranscription(params: TranscriptionConfig): Promise<void>;
+
   /**
    * @alpha
    * Used to define date range metadata in a media playlist.

diff --git a/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts b/packages/hms-video-store/src/analytics/stats/SubscribeStatsAnalytics.ts
@@ -56,7 +56,7 @@ export class SubscribeStatsAnalytics extends BaseStatsAnalytics {
       const getCalculatedJitterBufferDelay = (trackStats: HMSTrackStats) =>
         trackStats.jitterBufferDelay &&
         trackStats.jitterBufferEmittedCount &&
-        trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount;
+        (trackStats.jitterBufferDelay / trackStats.jitterBufferEmittedCount) * 1000;
 
       const calculatedJitterBufferDelay = getCalculatedJitterBufferDelay(trackStats);
 

diff --git a/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts b/packages/hms-video-store/src/audio-sink-manager/AudioSinkManager.ts
@@ -5,11 +5,11 @@ import { ErrorFactory } from '../error/ErrorFactory';
 import { HMSAction } from '../error/HMSAction';
 import { EventBus } from '../events/EventBus';
 import { HMSDeviceChangeEvent, HMSTrackUpdate, HMSUpdateListener } from '../interfaces';
+import { isMobile } from '../internal';
 import { HMSRemoteAudioTrack } from '../media/tracks';
 import { HMSRemotePeer } from '../sdk/models/peer';
 import { Store } from '../sdk/store';
 import HMSLogger from '../utils/logger';
-import { isMobile } from '../utils/support';
 import { sleep } from '../utils/timer-utils';
 
 /**
@@ -41,6 +41,7 @@ export class AudioSinkManager {
   private state = { ...INITIAL_STATE };
   private listener?: HMSUpdateListener;
   private timer: ReturnType<typeof setInterval> | null = null;
+  private autoUnpauseTimer: ReturnType<typeof setInterval> | null = null;
   private earpieceSelected = false;
 
   constructor(private store: Store, private deviceManager: DeviceManager, private eventBus: EventBus) {
@@ -49,6 +50,7 @@ export class AudioSinkManager {
     this.eventBus.audioTrackUpdate.subscribe(this.handleTrackUpdate);
     this.eventBus.deviceChange.subscribe(this.handleAudioDeviceChange);
     this.startPollingForDevices();
+    this.startPollingToCheckPausedAudio();
   }
 
   setListener(listener?: HMSUpdateListener) {
@@ -100,6 +102,10 @@ export class AudioSinkManager {
       clearInterval(this.timer);
       this.timer = null;
     }
+    if (this.autoUnpauseTimer) {
+      clearInterval(this.autoUnpauseTimer);
+      this.autoUnpauseTimer = null;
+    }
     this.eventBus.audioTrackAdded.unsubscribe(this.handleTrackAdd);
     this.eventBus.audioTrackRemoved.unsubscribe(this.handleTrackRemove);
     this.eventBus.audioTrackUpdate.unsubscribe(this.handleTrackUpdate);
@@ -109,24 +115,11 @@ export class AudioSinkManager {
   }
 
   private handleAudioPaused = async (event: any) => {
-    const audioEl = event.target as HTMLAudioElement;
-    //@ts-ignore
-    const track = audioEl.srcObject?.getAudioTracks()[0];
-    if (!track?.enabled) {
-      // No need to play if already disabled
-      return;
-    }
-    // this means the audio paused because of external factors(headset removal)
+    // this means the audio paused because of external factors(headset removal, incoming phone call)
     HMSLogger.d(this.TAG, 'Audio Paused', event.target.id);
     const audioTrack = this.store.getTrackById(event.target.id);
     if (audioTrack) {
-      if (isMobile()) {
-        // Play after a delay since mobile devices don't call onDevice change event
-        await sleep(500);
-        this.playAudioFor(audioTrack as HMSRemoteAudioTrack);
-      } else {
-        this.autoPausedTracks.add(audioTrack as HMSRemoteAudioTrack);
-      }
+      this.autoPausedTracks.add(audioTrack as HMSRemoteAudioTrack);
     }
   };
 
@@ -272,6 +265,14 @@ export class AudioSinkManager {
     }
   };
 
+  private startPollingToCheckPausedAudio = () => {
+    if (isMobile()) {
+      this.autoUnpauseTimer = setInterval(() => {
+        this.unpauseAudioTracks();
+      }, 5000);
+    }
+  };
+
   private startPollingForDevices = () => {
     // device change supported, no polling needed
     if ('ondevicechange' in navigator.mediaDevices) {
@@ -281,6 +282,7 @@ export class AudioSinkManager {
       (async () => {
         await this.deviceManager.init(true, false);
         await this.autoSelectAudioOutput();
+        this.unpauseAudioTracks();
       })();
     }, 5000);
   };

diff --git a/packages/hms-video-store/src/interfaces/hms.ts b/packages/hms-video-store/src/interfaces/hms.ts
@@ -12,6 +12,7 @@ import { HMSHLS, HMSRecording, HMSRTMP, HMSTranscriptionInfo } from './room';
 import { RTMPRecordingConfig } from './rtmp-recording-config';
 import { HMSInteractivityCenter, HMSSessionStore } from './session-store';
 import { HMSScreenShareConfig } from './track-settings';
+import { TranscriptionConfig } from './transcription-config';
 import { HMSAudioListener, HMSConnectionQualityListener, HMSUpdateListener } from './update-listener';
 import { HMSAnalyticsLevel } from '../analytics/AnalyticsEventLevel';
 import { IAudioOutputManager } from '../device-manager/AudioOutputManager';
@@ -61,6 +62,8 @@ export interface HMSInterface {
    */
   startHLSStreaming(params?: HLSConfig): Promise<void>;
   stopHLSStreaming(params?: HLSConfig): Promise<void>;
+  startTranscription(params: TranscriptionConfig): Promise<void>;
+  stopTranscription(params: TranscriptionConfig): Promise<void>;
   getRecordingState(): HMSRecording | undefined;
   getRTMPState(): HMSRTMP | undefined;
   getHLSState(): HMSHLS | undefined;

diff --git a/packages/hms-video-store/src/interfaces/index.ts b/packages/hms-video-store/src/interfaces/index.ts
@@ -19,3 +19,4 @@ export * from './webrtc-stats';
 export * from './framework-info';
 export * from './get-token';
 export * from './session-store';
+export * from './transcription-config';
diff --git a/packages/hms-video-store/src/interfaces/room.ts b/packages/hms-video-store/src/interfaces/room.ts
@@ -123,6 +123,7 @@ export interface HLSVariant {
 Transcription related details
 */
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',
@@ -133,4 +134,9 @@ export enum HMSTranscriptionMode {
 export interface HMSTranscriptionInfo {
   state?: HMSTranscriptionState;
   mode?: HMSTranscriptionMode;
+  initialised_at?: Date;
+  started_at?: Date;
+  updated_at?: Date;
+  stopped_at?: Date;
+  error?: HMSException;
 }
diff --git a/packages/hms-video-store/src/interfaces/session-store/interactivity-center.ts b/packages/hms-video-store/src/interfaces/session-store/interactivity-center.ts
@@ -20,6 +20,7 @@ export interface HMSInteractivityCenter {
   stopPoll(pollID: string): Promise<void>;
   addResponsesToPoll(pollID: string, responses: HMSPollQuestionResponseCreateParams[]): Promise<void>;
   fetchLeaderboard(pollID: string, offset: number, count: number): Promise<HMSQuizLeaderboardResponse>;
+  getPollResponses(poll: HMSPoll, self: boolean): Promise<void>;
   getPolls(): Promise<HMSPoll[]>;
   /** @alpha */
   whiteboard: HMSWhiteboardInteractivityCenter;

diff --git a/packages/hms-video-store/src/interfaces/transcription-config.ts b/packages/hms-video-store/src/interfaces/transcription-config.ts
@@ -0,0 +1,5 @@
+import { HMSTranscriptionMode } from './room';
+
+export interface TranscriptionConfig {
+  mode: HMSTranscriptionMode;
+}
diff --git a/packages/hms-video-store/src/interfaces/update-listener.ts b/packages/hms-video-store/src/interfaces/update-listener.ts
@@ -17,6 +17,7 @@ export enum HMSRoomUpdate {
   SERVER_RECORDING_STATE_UPDATED = 'SERVER_RECORDING_STATE_UPDATED',
   RTMP_STREAMING_STATE_UPDATED = 'RTMP_STREAMING_STATE_UPDATED',
   HLS_STREAMING_STATE_UPDATED = 'HLS_STREAMING_STATE_UPDATED',
+  TRANSCRIPTION_STATE_UPDATED = 'TRANSCRIPTION_STATE_UPDATED',
   ROOM_PEER_COUNT_UPDATED = 'ROOM_PEER_COUNT_UPDATED',
 }
 
@@ -54,7 +55,6 @@ export enum HMSPollsUpdate {
   POLL_STOPPED,
   POLLS_LIST,
   POLL_STATS_UPDATED,
-  // POLL_LEADERBOARD_SHARED,
 }
 
 export interface HMSAudioListener {

diff --git a/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts b/packages/hms-video-store/src/media/tracks/HMSVideoTrack.ts
@@ -69,6 +69,7 @@ export class HMSVideoTrack extends HMSTrack {
       if (existingTrack?.id === track.id) {
         if (!existingTrack.muted && existingTrack.readyState === 'live') {
           // it's already attached, attaching again would just cause flickering
+          this.reTriggerPlay({ videoElement, stream: srcObject });
           return;
         } else {
           this.reduceSinkCount();
@@ -77,10 +78,22 @@ export class HMSVideoTrack extends HMSTrack {
         this.reduceSinkCount();
       }
     }
-    videoElement.srcObject = new MediaStream([track]);
+    const stream = new MediaStream([track]);
+    videoElement.srcObject = stream;
+    this.reTriggerPlay({ videoElement, stream });
     this.sinkCount++;
   }
 
+  private reTriggerPlay = ({ videoElement, stream }: { videoElement: HTMLVideoElement; stream: MediaStream }) => {
+    setTimeout(() => {
+      if (videoElement.paused) {
+        // This is needed for safari and firefox to work properly
+        videoElement.srcObject = stream;
+        videoElement.play().catch(console.error);
+      }
+    }, 0);
+  };
+
   private reduceSinkCount() {
     if (this.sinkCount > 0) {
       this.sinkCount--;

diff --git a/packages/hms-video-store/src/media/tracks/VideoElementManager.ts b/packages/hms-video-store/src/media/tracks/VideoElementManager.ts
@@ -5,7 +5,7 @@ import { HMSLocalVideoTrack, HMSRemoteVideoTrack } from '../../internal';
 import { HMSIntersectionObserver } from '../../utils/intersection-observer';
 import HMSLogger from '../../utils/logger';
 import { HMSResizeObserver } from '../../utils/resize-observer';
-import { isBrowser } from '../../utils/support';
+import { isBrowser, isMobile } from '../../utils/support';
 
 /**
  * This class is to manager video elements for video tracks.
@@ -45,6 +45,7 @@ export class VideoElementManager {
     this.init();
     HMSLogger.d(this.TAG, `Adding video element for ${this.track}`, this.id);
     this.videoElements.add(videoElement);
+    videoElement.addEventListener('pause', this.resumeVideoPlayback);
     if (this.videoElements.size >= 10) {
       HMSLogger.w(
         this.TAG,
@@ -72,6 +73,7 @@ export class VideoElementManager {
   removeVideoElement(videoElement: HTMLVideoElement): void {
     this.track.removeSink(videoElement);
     this.videoElements.delete(videoElement);
+    videoElement.removeEventListener('pause', this.resumeVideoPlayback);
     this.entries.delete(videoElement);
     this.resizeObserver?.unobserve(videoElement);
     this.intersectionObserver?.unobserve(videoElement);
@@ -82,6 +84,14 @@ export class VideoElementManager {
     return Array.from(this.videoElements);
   }
 
+  private resumeVideoPlayback = (e: Event) => {
+    const element = e.target as HTMLVideoElement;
+    if (!document.hidden && isMobile() && element.paused) {
+      // add sink again to play the video specially in safari iOS
+      this.track.addSink(element);
+    }
+  };
+
   private init() {
     if (isBrowser) {
       this.resizeObserver = HMSResizeObserver;
@@ -172,6 +182,7 @@ export class VideoElementManager {
   cleanup = () => {
     this.videoElements.forEach(videoElement => {
       videoElement.srcObject = null;
+      videoElement.removeEventListener('pause', this.resumeVideoPlayback);
       this.resizeObserver?.unobserve(videoElement);
       this.intersectionObserver?.unobserve(videoElement);
     });

diff --git a/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts b/packages/hms-video-store/src/notification-manager/HMSNotificationMethod.ts
@@ -28,6 +28,7 @@ export enum HMSNotificationMethod {
   RTMP_UPDATE = 'on-rtmp-update',
   RECORDING_UPDATE = 'on-record-update',
   HLS_UPDATE = 'on-hls-update',
+  TRANSCRIPTION_UPDATE = 'on-transcription-update',
   METADATA_CHANGE = 'on-metadata-change',
   POLL_START = 'on-poll-start',
   POLL_STOP = 'on-poll-stop',

diff --git a/packages/hms-video-store/src/notification-manager/HMSNotifications.ts b/packages/hms-video-store/src/notification-manager/HMSNotifications.ts
@@ -62,6 +62,7 @@ export enum HMSStreamingState {
 }
 
 export enum HMSTranscriptionState {
+  INITIALISED = 'initialised',
   STARTED = 'started',
   STOPPED = 'stopped',
   FAILED = 'failed',
@@ -133,6 +134,12 @@ export interface PeerNotification {
 export interface TranscriptionNotification {
   state?: HMSTranscriptionState;
   mode?: HMSTranscriptionMode;
+  initialised_at?: number;
+  started_at?: number;
+  updated_at?: number;
+  stopped_at?: number;
+  peer?: PeerNotificationInfo;
+  error?: ServerError;
 }
 
 export interface RoomState {

diff --git a/packages/hms-video-store/src/notification-manager/managers/PollsManager.ts b/packages/hms-video-store/src/notification-manager/managers/PollsManager.ts
@@ -88,8 +88,6 @@ export class PollsManager {
       }
 
       this.updatePollResult(savedPoll, updatedPoll);
-      await this.updatePollResponses(savedPoll, true);
-
       updatedPolls.push(savedPoll);
     }
 

diff --git a/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts b/packages/hms-video-store/src/notification-manager/managers/RoomUpdateManager.ts
@@ -59,6 +59,9 @@ export class RoomUpdateManager {
       case HMSNotificationMethod.HLS_UPDATE:
         this.updateHLSStatus(notification as HLSNotification);
         break;
+      case HMSNotificationMethod.TRANSCRIPTION_UPDATE:
+        this.handleTranscriptionStatus([notification as TranscriptionNotification]);
+        break;
       default:
         break;
     }
@@ -129,6 +132,11 @@ export class RoomUpdateManager {
       return {
         state: transcription.state,
         mode: transcription.mode,
+        initialised_at: convertDateNumToDate(transcription.initialised_at),
+        started_at: convertDateNumToDate(transcription.started_at),
+        stopped_at: convertDateNumToDate(transcription.stopped_at),
+        updated_at: convertDateNumToDate(transcription.updated_at),
+        error: this.toSdkError(transcription?.error),
       };
     });
   }
@@ -200,6 +208,15 @@ export class RoomUpdateManager {
     this.listener?.onRoomUpdate(HMSRoomUpdate.HLS_STREAMING_STATE_UPDATED, room);
   }
 
+  private handleTranscriptionStatus(notification: TranscriptionNotification[]) {
+    const room = this.store.getRoom();
+    if (!room) {
+      HMSLogger.w(this.TAG, 'on transcription - room not present');
+      return;
+    }
+    room.transcriptions = this.addTranscriptionDetail(notification) || [];
+    this.listener?.onRoomUpdate(HMSRoomUpdate.TRANSCRIPTION_STATE_UPDATED, room);
+  }
   private convertHls(hlsNotification?: HLSNotification) {
     // only checking for zeroth variant intialized
     const isInitialised =

diff --git a/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts b/packages/hms-video-store/src/reactive-store/HMSSDKActions.ts
@@ -701,6 +701,14 @@ export class HMSSDKActions<T extends HMSGenericTypes = { sessionStore: Record<st
     await this.sdk.stopHLSStreaming(params);
   }
 
+  async startTranscription(params: sdkTypes.TranscriptionConfig) {
+    await this.sdk.startTranscription(params);
+  }
+
+  async stopTranscription(params: sdkTypes.TranscriptionConfig): Promise<void> {
+    await this.sdk.stopTranscription(params);
+  }
+
   async sendHLSTimedMetadata(metadataList: sdkTypes.HLSTimedMetadata[]): Promise<void> {
     await this.sdk.sendHLSTimedMetadata(metadataList);
   }

diff --git a/packages/hms-video-store/src/reactive-store/adapter.ts b/packages/hms-video-store/src/reactive-store/adapter.ts
@@ -139,10 +139,11 @@ export class SDKToHMS {
   }
 
   static convertRoom(sdkRoom: sdkTypes.HMSRoom, sdkLocalPeerId?: string): Partial<HMSRoom> {
-    const { recording, rtmp, hls } = SDKToHMS.convertRecordingStreamingState(
-      sdkRoom?.recording,
-      sdkRoom?.rtmp,
-      sdkRoom?.hls,
+    const { recording, rtmp, hls, transcriptions } = SDKToHMS.convertRecordingStreamingState(
+      sdkRoom.recording,
+      sdkRoom.rtmp,
+      sdkRoom.hls,
+      sdkRoom.transcriptions,
     );
     return {
       id: sdkRoom.id,
@@ -151,6 +152,7 @@ export class SDKToHMS {
       recording,
       rtmp,
       hls,
+      transcriptions,
       sessionId: sdkRoom.sessionId,
       startedAt: sdkRoom.startedAt,
       joinedAt: sdkRoom.joinedAt,

diff --git a/packages/hms-video-store/src/reactive-store/common/mapping.ts b/packages/hms-video-store/src/reactive-store/common/mapping.ts
@@ -31,5 +31,4 @@ export const POLL_NOTIFICATION_TYPES: PollNotificationMap = {
   [sdkTypes.HMSPollsUpdate.POLL_STOPPED]: HMSNotificationTypes.POLL_STOPPED,
   [sdkTypes.HMSPollsUpdate.POLL_STATS_UPDATED]: HMSNotificationTypes.POLL_VOTES_UPDATED,
   [sdkTypes.HMSPollsUpdate.POLLS_LIST]: HMSNotificationTypes.POLLS_LIST,
-  // [sdkTypes.HMSPollsUpdate.POLL_LEADERBOARD_SHARED]: HMSNotificationTypes.POLL_LEADERBOARD_SHARED,
 };
-Original file line number
+Diff line change
@@ Expand Up / @@ -88,8 +88,6 @@ export class PollsManager { @@
           }
           this.updatePollResult(savedPoll, updatedPoll);
-          await this.updatePollResponses(savedPoll, true);
           updatedPolls.push(savedPoll);
         }
@@ Expand Down @@