feat: manual video quality selection (#1486)

Adds APIs to manually override incoming video quality: ## Client ### `call.setPreferredIncomingVideoResolution(resolution?: VideoDimension, sessionIds?: string[])` Specify preference for incoming video resolution. The preference will be matched as close as possible, but actual resolution will depend on the video source quality and client network conditions. Will enable incoming video, if previously disabled. Passing `undefined` resolution clears previously set preferences. Optionally specify session ids of the participants this preference has effect on. Affects all participants by default. ### `call.setIncomingVideoEnabled(enabled: boolean)` Enables or disables incoming video from all remote call participants, and clears any preference for preferred resolution. ## React SDK New call state hook added: ### `useIncomingVideoSettings()` Returns incoming video settings for the current call, including global and per-participant manual video quality overrides. ## Demo App Video quality selector added to the settings modal in `react-dogfood`. On Pronto, it's also visible in call controls: ![](https://github.com/user-attachments/assets/8cf111d5-2924-4067-9aca-59f51dfb1a8e) ![](https://github.com/user-attachments/assets/eac779c0-d120-402d-b7b9-ef3033b56d1e) ## Internal APIs updated ### `callState.updateParticipantTracks` Call class no longer has a slightly incorrectly named `updateSubscriptionsPartial` method. We want to avoid the Call class dealing with subscriptions at all - this is a domain of the DynascaleManager. Instead, the new `updateParticipantTracks` should be used on the CallState object. ### Disabling <Video /> New `enabled` prop has been added to the core Video component. Setting `enabled={false}` allows to force fallback instead of video, even if participant is publishing a video track. --------- Co-authored-by: Santhosh Vaiyapuri <[email protected]>
GetStream · Oct 2, 2024 · 3a754af · 3a754af
1 parent 0a2b748
commit 3a754af
Show file tree

Hide file tree

Showing 33 changed files with 940 additions and 304 deletions.
diff --git a/...nt/docusaurus/docs/javascript/10-advanced/12-manual-video-quality-selection.mdx b/...nt/docusaurus/docs/javascript/10-advanced/12-manual-video-quality-selection.mdx
@@ -0,0 +1,59 @@
+---
+id: manual-video-quality-selection
+title: Manual Video Quality Selection
+---
+
+By default, our SDK chooses the incoming video quality that best matches the size of a video element for a given participant. It makes less sense to waste bandwidth receiving Full HD video when it's going to be displayed in a 320 by 240 pixel rectangle.
+
+However, it's still possible to override this behavior and manually request higher resolution video for better quality, or lower resolution to save bandwidth. It's also possible to disable incoming video altogether for an audio-only experience.
+
+## Overriding Preferred Resolution
+
+To override the preferred incoming video resolution, use the `call.setPreferredIncomingVideoResolution` method:
+
+```js
+await call.setPreferredIncomingVideoResolution({ width: 640, height: 480 });
+```
+
+:::note
+Actual incoming video quality depends on a number of factors, such as the quality of the source video, and network conditions. Manual video quality selection allows you to specify your preference, while the actual resolution is automatically selected from the available resolutions to match that preference as closely as possible.
+:::
+
+It's also possible to override the incoming video resolution for only a selected subset of call participants. The `call.setPreferredIncomingVideoResolution` method optionally takes an array of participant session identifiers as its second argument. Session identifiers can be obtained from the call participant state:
+
+```js
+const [firstParticipant, secondParticipant] = call.state.participants;
+// Set preferred incoming video resolution for the first two participants only:
+await call.setPreferredIncomingVideoResolution({ width: 640, height: 480 }, [
+  [firstParticipant.sessionId, secondParticipant.sessionId],
+]);
+```
+
+Calling this method will enable incoming video for the selected participants if it was previously disabled.
+
+To clear a previously set preference, pass `undefined` instead of resolution:
+
+```js
+// Clear resolution preference for selected participants:
+await call.setPreferredIncomingVideoResolution(undefined, [
+  participant.sessionId,
+]);
+// Clear resolution preference for all participants:
+await call.setPreferredIncomingVideoResolution(undefined);
+```
+
+## Disabling Incoming Video
+
+To completely disable incoming video (either to save data, or for an audio-only experience), use the `call.setIncomingVideoEnabled` method:
+
+```js
+await call.setIncomingVideoEnabled(false);
+```
+
+To enable incoming video again, pass `true` as an argument:
+
+```js
+await call.setIncomingVideoEnabled(true);
+```
+
+Calling this method will clear the previously set resolution preferences.
diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts
@@ -8,11 +8,6 @@ import {
 } from './rtc';
 import { muteTypeToTrackType } from './rtc/helpers/tracks';
 import { toRtcConfiguration } from './rtc/helpers/rtcConfiguration';
-import {
-  hasScreenShare,
-  hasScreenShareAudio,
-  hasVideo,
-} from './helpers/participantUtils';
 import {
   registerEventHandlers,
   registerRingingCallEventHandlers,
@@ -79,30 +74,19 @@ import type {
   UpdateCallResponse,
   UpdateUserPermissionsRequest,
   UpdateUserPermissionsResponse,
+  VideoResolution,
 } from './gen/coordinator';
 import { OwnCapability } from './gen/coordinator';
 import {
   AudioTrackType,
   CallConstructor,
   CallLeaveOptions,
-  DebounceType,
   JoinCallData,
   PublishOptions,
-  StreamVideoParticipant,
-  StreamVideoParticipantPatches,
-  SubscriptionChanges,
   TrackMuteType,
   VideoTrackType,
 } from './types';
-import {
-  BehaviorSubject,
-  debounce,
-  map,
-  Subject,
-  takeWhile,
-  timer,
-} from 'rxjs';
-import { TrackSubscriptionDetails } from './gen/video/sfu/signal_rpc/signal';
+import { BehaviorSubject, Subject, takeWhile } from 'rxjs';
 import {
   ReconnectDetails,
   VideoLayerSetting,
@@ -196,7 +180,7 @@ export class Call {
   /**
    * The DynascaleManager instance.
    */
-  readonly dynascaleManager = new DynascaleManager(this);
+  readonly dynascaleManager: DynascaleManager;
 
   subscriber?: Subscriber;
   publisher?: Publisher;
@@ -218,11 +202,6 @@ export class Call {
    */
   private readonly dispatcher = new Dispatcher();
 
-  private trackSubscriptionsSubject = new BehaviorSubject<{
-    type: DebounceType;
-    data: TrackSubscriptionDetails[];
-  }>({ type: DebounceType.MEDIUM, data: [] });
-
   private statsReporter?: StatsReporter;
   private sfuStatsReporter?: SfuStatsReporter;
   private dropTimeout: ReturnType<typeof setTimeout> | undefined;
@@ -304,6 +283,7 @@ export class Call {
     this.microphone = new MicrophoneManager(this);
     this.speaker = new SpeakerManager(this);
     this.screenShare = new ScreenShareManager(this);
+    this.dynascaleManager = new DynascaleManager(this.state, this.speaker);
   }
 
   private async setup() {
@@ -321,19 +301,6 @@ export class Call {
       this.registerEffects();
       this.registerReconnectHandlers();
 
-      this.leaveCallHooks.add(
-        createSubscription(
-          this.trackSubscriptionsSubject.pipe(
-            debounce((v) => timer(v.type)),
-            map((v) => v.data),
-          ),
-          (subscriptions) =>
-            this.sfuClient?.updateSubscriptions(subscriptions).catch((err) => {
-              this.logger('debug', `Failed to update track subscriptions`, err);
-            }),
-        ),
-      );
-
       if (this.state.callingState === CallingState.LEFT) {
         this.state.setCallingState(CallingState.IDLE);
       }
@@ -582,6 +549,7 @@ export class Call {
 
       await this.sfuClient?.leaveAndClose(reason);
       this.sfuClient = undefined;
+      this.dynascaleManager.setSfuClient(undefined);
 
       this.state.setCallingState(CallingState.LEFT);
 
@@ -812,6 +780,7 @@ export class Call {
           })
         : previousSfuClient;
     this.sfuClient = sfuClient;
+    this.dynascaleManager.setSfuClient(sfuClient);
 
     const clientDetails = getClientDetails();
     // we don't need to send JoinRequest if we are re-using an existing healthy SFU client
@@ -904,11 +873,10 @@ export class Call {
     const strategy = this.reconnectStrategy;
     const performingRejoin = strategy === WebsocketReconnectStrategy.REJOIN;
     const announcedTracks = this.publisher?.getAnnouncedTracks() || [];
-    const subscribedTracks = getCurrentValue(this.trackSubscriptionsSubject);
     return {
       strategy,
       announcedTracks,
-      subscriptions: subscribedTracks.data || [],
+      subscriptions: this.dynascaleManager.trackSubscriptions,
       reconnectAttempt: this.reconnectAttempts,
       fromSfuId: migratingFromSfuId || '',
       previousSessionId: performingRejoin ? previousSessionId || '' : '',
@@ -1357,7 +1325,7 @@ export class Call {
   private restoreSubscribedTracks = () => {
     const { remoteParticipants } = this.state;
     if (remoteParticipants.length <= 0) return;
-    this.updateSubscriptions(remoteParticipants, DebounceType.FAST);
+    this.dynascaleManager.applyTrackSubscriptions(undefined);
   };
 
   /**
@@ -1521,87 +1489,6 @@ export class Call {
     });
   };
 
-  /**
-   * Update track subscription configuration for one or more participants.
-   * You have to create a subscription for each participant for all the different kinds of tracks you want to receive.
-   * You can only subscribe for tracks after the participant started publishing the given kind of track.
-   *
-   * @param trackType the kind of subscription to update.
-   * @param changes the list of subscription changes to do.
-   * @param type the debounce type to use for the update.
-   */
-  updateSubscriptionsPartial = (
-    trackType: VideoTrackType,
-    changes: SubscriptionChanges,
-    type: DebounceType = DebounceType.SLOW,
-  ) => {
-    const participants = this.state.updateParticipants(
-      Object.entries(changes).reduce<StreamVideoParticipantPatches>(
-        (acc, [sessionId, change]) => {
-          if (change.dimension) {
-            change.dimension.height = Math.ceil(change.dimension.height);
-            change.dimension.width = Math.ceil(change.dimension.width);
-          }
-          const prop: keyof StreamVideoParticipant | undefined =
-            trackType === 'videoTrack'
-              ? 'videoDimension'
-              : trackType === 'screenShareTrack'
-                ? 'screenShareDimension'
-                : undefined;
-          if (prop) {
-            acc[sessionId] = {
-              [prop]: change.dimension,
-            };
-          }
-          return acc;
-        },
-        {},
-      ),
-    );
-
-    this.updateSubscriptions(participants, type);
-  };
-
-  private updateSubscriptions = (
-    participants: StreamVideoParticipant[],
-    type: DebounceType = DebounceType.SLOW,
-  ) => {
-    const subscriptions: TrackSubscriptionDetails[] = [];
-    for (const p of participants) {
-      // we don't want to subscribe to our own tracks
-      if (p.isLocalParticipant) continue;
-
-      // NOTE: audio tracks don't have to be requested explicitly
-      // as the SFU will implicitly subscribe us to all of them,
-      // once they become available.
-      if (p.videoDimension && hasVideo(p)) {
-        subscriptions.push({
-          userId: p.userId,
-          sessionId: p.sessionId,
-          trackType: TrackType.VIDEO,
-          dimension: p.videoDimension,
-        });
-      }
-      if (p.screenShareDimension && hasScreenShare(p)) {
-        subscriptions.push({
-          userId: p.userId,
-          sessionId: p.sessionId,
-          trackType: TrackType.SCREEN_SHARE,
-          dimension: p.screenShareDimension,
-        });
-      }
-      if (hasScreenShareAudio(p)) {
-        subscriptions.push({
-          userId: p.userId,
-          sessionId: p.sessionId,
-          trackType: TrackType.SCREEN_SHARE_AUDIO,
-        });
-      }
-    }
-    // schedule update
-    this.trackSubscriptionsSubject.next({ type, data: subscriptions });
-  };
-
   /**
    * Will enhance the reported stats with additional participant-specific information (`callStatsReport$` state [store variable](./StreamVideoClient.md/#readonlystatestore)).
    * This is usually helpful when detailed stats for a specific participant are needed.
@@ -2387,4 +2274,41 @@ export class Call {
       imageElement.removeEventListener('error', handleError);
     };
   };
+
+  /**
+   * Specify preference for incoming video resolution. The preference will
+   * be matched as close as possible, but actual resolution will depend
+   * on the video source quality and client network conditions. Will enable
+   * incoming video, if previously disabled.
+   *
+   * @param resolution preferred resolution, or `undefined` to clear preference
+   * @param sessionIds optionally specify session ids of the participants this
+   * preference has effect on. Affects all participants by default.
+   */
+  setPreferredIncomingVideoResolution = (
+    resolution: VideoResolution | undefined,
+    sessionIds?: string[],
+  ) => {
+    this.dynascaleManager.setVideoTrackSubscriptionOverrides(
+      resolution
+        ? {
+            enabled: true,
+            dimension: resolution,
+          }
+        : undefined,
+      sessionIds,
+    );
+    this.dynascaleManager.applyTrackSubscriptions();
+  };
+
+  /**
+   * Enables or disables incoming video from all remote call participants,
+   * and removes any preference for preferred resolution.
+   */
+  setIncomingVideoEnabled = (enabled: boolean) => {
+    this.dynascaleManager.setVideoTrackSubscriptionOverrides(
+      enabled ? undefined : { enabled: false },
+    );
+    this.dynascaleManager.applyTrackSubscriptions();
+  };
 }