feat: manual video quality selection

packages/client/docusaurus/docs/javascript/10-advanced/12-manual-video-quality-selection.mdx

@@ -0,0 +1,59 @@
+---
+id: manual-video-quality-selection
+title: Manual Video Quality Selection
+---
+
+By default, our SDK chooses the incoming video quality that best matches the size of a video element for a given participant. It makes less sense to waste bandwidth receiving Full HD video when it's going to be displayed in a 320 by 240 pixel rectangle.
+
+However, it's still possible to override this behavior and manually request higher resolution video for better quality, or lower resolution to save bandwidth. It's also possible to disable incoming video altogether for an audio-only experience.
+
+## Overriding Preferred Resolution
+
+To override the preferred incoming video resolution, use the `call.setPreferredIncomingVideoResolution` method:
+
+```js
+await call.setPreferredIncomingVideoResolution({ width: 640, height: 480 });
+```
+
+:::note
+Actual incoming video quality depends on a number of factors, such as the quality of the source video, and network conditions. Manual video quality selection allows you to specify your preference, while the actual resolution is automatically selected from the available resolutions to match that preference as closely as possible.
+:::
+
+It's also possible to override the incoming video resolution for only a selected subset of call participants. The `call.setPreferredIncomingVideoResolution` method optionally takes an array of participant session identifiers as its second argument. Session identifiers can be obtained from the call participant state:
+
+```js
+const [firstParticipant, secondParticipant] = call.state.participants;
+// Set preferred incoming video resolution for the first two participants only:
+await call.setPreferredIncomingVideoResolution({ width: 640, height: 480 }, [
+  [firstParticipant.sessionId, secondParticipant.sessionId],
+]);
+```
+
+Calling this method will enable incoming video for the selected participants if it was previously disabled.
+
+To clear a previously set preference, pass `undefined` instead of resolution:
+
+```js
+// Clear resolution preference for selected participants:
+await call.setPreferredIncomingVideoResolution(undefined, [
+  participant.sessionId,
+]);
+// Clear resolution preference for all participants:
+await call.setPreferredIncomingVideoResolution(undefined);
+```
+
+## Disabling Incoming Video
+
+To completely disable incoming video (either to save data, or for an audio-only experience), use the `call.setIncomingVideoEnabled` method:
+
+```js
+await call.setIncomingVideoEnabled(false);
+```
+
+To enable incoming video again, pass `true` as an argument:
+
+```js
+await call.setIncomingVideoEnabled(true);
+```
+
+Calling this method will clear the previously set resolution preferences.

packages/client/src/Call.ts

@@ -8,11 +8,6 @@ import {
 } from './rtc';
 import { muteTypeToTrackType } from './rtc/helpers/tracks';
 import { toRtcConfiguration } from './rtc/helpers/rtcConfiguration';
-import {
-  hasScreenShare,
-  hasScreenShareAudio,
-  hasVideo,
-} from './helpers/participantUtils';
 import {
   registerEventHandlers,
   registerRingingCallEventHandlers,
@@ -79,30 +74,19 @@ import type {
   UpdateCallResponse,
   UpdateUserPermissionsRequest,
   UpdateUserPermissionsResponse,
+  VideoResolution,
 } from './gen/coordinator';
 import { OwnCapability } from './gen/coordinator';
 import {
   AudioTrackType,
   CallConstructor,
   CallLeaveOptions,
-  DebounceType,
   JoinCallData,
   PublishOptions,
-  StreamVideoParticipant,
-  StreamVideoParticipantPatches,
-  SubscriptionChanges,
   TrackMuteType,
   VideoTrackType,
 } from './types';
-import {
-  BehaviorSubject,
-  debounce,
-  map,
-  Subject,
-  takeWhile,
-  timer,
-} from 'rxjs';
-import { TrackSubscriptionDetails } from './gen/video/sfu/signal_rpc/signal';
+import { BehaviorSubject, Subject, takeWhile } from 'rxjs';
 import {
   ReconnectDetails,
   VideoLayerSetting,
@@ -196,7 +180,7 @@ export class Call {
   /**
    * The DynascaleManager instance.
    */
-  readonly dynascaleManager = new DynascaleManager(this);
+  readonly dynascaleManager: DynascaleManager;
 
   subscriber?: Subscriber;
   publisher?: Publisher;
@@ -218,11 +202,6 @@ export class Call {
    */
   private readonly dispatcher = new Dispatcher();
 
-  private trackSubscriptionsSubject = new BehaviorSubject<{
-    type: DebounceType;
-    data: TrackSubscriptionDetails[];
-  }>({ type: DebounceType.MEDIUM, data: [] });
-
   private statsReporter?: StatsReporter;
   private sfuStatsReporter?: SfuStatsReporter;
   private dropTimeout: ReturnType<typeof setTimeout> | undefined;
@@ -304,6 +283,7 @@ export class Call {
     this.microphone = new MicrophoneManager(this);
     this.speaker = new SpeakerManager(this);
     this.screenShare = new ScreenShareManager(this);
+    this.dynascaleManager = new DynascaleManager(this.state, this.speaker);
   }
 
   private async setup() {
@@ -321,19 +301,6 @@ export class Call {
       this.registerEffects();
       this.registerReconnectHandlers();
 
-      this.leaveCallHooks.add(
-        createSubscription(
-          this.trackSubscriptionsSubject.pipe(
-            debounce((v) => timer(v.type)),
-            map((v) => v.data),
-          ),
-          (subscriptions) =>
-            this.sfuClient?.updateSubscriptions(subscriptions).catch((err) => {
-              this.logger('debug', `Failed to update track subscriptions`, err);
-            }),
-        ),
-      );
-
       if (this.state.callingState === CallingState.LEFT) {
         this.state.setCallingState(CallingState.IDLE);
       }
@@ -582,6 +549,7 @@ export class Call {
 
       await this.sfuClient?.leaveAndClose(reason);
       this.sfuClient = undefined;
+      this.dynascaleManager.setSfuClient(undefined);
 
       this.state.setCallingState(CallingState.LEFT);
 
@@ -812,6 +780,7 @@ export class Call {
           })
         : previousSfuClient;
     this.sfuClient = sfuClient;
+    this.dynascaleManager.setSfuClient(sfuClient);
 
     const clientDetails = getClientDetails();
     // we don't need to send JoinRequest if we are re-using an existing healthy SFU client
@@ -904,11 +873,10 @@ export class Call {
     const strategy = this.reconnectStrategy;
     const performingRejoin = strategy === WebsocketReconnectStrategy.REJOIN;
     const announcedTracks = this.publisher?.getAnnouncedTracks() || [];
-    const subscribedTracks = getCurrentValue(this.trackSubscriptionsSubject);
     return {
       strategy,
       announcedTracks,
-      subscriptions: subscribedTracks.data || [],
+      subscriptions: this.dynascaleManager.trackSubscriptions,
       reconnectAttempt: this.reconnectAttempts,
       fromSfuId: migratingFromSfuId || '',
       previousSessionId: performingRejoin ? previousSessionId || '' : '',
@@ -1357,7 +1325,7 @@ export class Call {
   private restoreSubscribedTracks = () => {
     const { remoteParticipants } = this.state;
     if (remoteParticipants.length <= 0) return;
-    this.updateSubscriptions(remoteParticipants, DebounceType.FAST);
+    this.dynascaleManager.applyTrackSubscriptions(undefined);
   };
 
   /**
@@ -1521,87 +1489,6 @@ export class Call {
     });
   };
 
-  /**
-   * Update track subscription configuration for one or more participants.
-   * You have to create a subscription for each participant for all the different kinds of tracks you want to receive.
-   * You can only subscribe for tracks after the participant started publishing the given kind of track.
-   *
-   * @param trackType the kind of subscription to update.
-   * @param changes the list of subscription changes to do.
-   * @param type the debounce type to use for the update.
-   */
-  updateSubscriptionsPartial = (
-    trackType: VideoTrackType,
-    changes: SubscriptionChanges,
-    type: DebounceType = DebounceType.SLOW,
-  ) => {
-    const participants = this.state.updateParticipants(
-      Object.entries(changes).reduce<StreamVideoParticipantPatches>(
-        (acc, [sessionId, change]) => {
-          if (change.dimension) {
-            change.dimension.height = Math.ceil(change.dimension.height);
-            change.dimension.width = Math.ceil(change.dimension.width);
-          }
-          const prop: keyof StreamVideoParticipant | undefined =
-            trackType === 'videoTrack'
-              ? 'videoDimension'
-              : trackType === 'screenShareTrack'
-                ? 'screenShareDimension'
-                : undefined;
-          if (prop) {
-            acc[sessionId] = {
-              [prop]: change.dimension,
-            };
-          }
-          return acc;
-        },
-        {},
-      ),
-    );
-
-    this.updateSubscriptions(participants, type);
-  };
-
-  private updateSubscriptions = (
-    participants: StreamVideoParticipant[],
-    type: DebounceType = DebounceType.SLOW,
-  ) => {
-    const subscriptions: TrackSubscriptionDetails[] = [];
-    for (const p of participants) {
-      // we don't want to subscribe to our own tracks
-      if (p.isLocalParticipant) continue;
-
-      // NOTE: audio tracks don't have to be requested explicitly
-      // as the SFU will implicitly subscribe us to all of them,
-      // once they become available.
-      if (p.videoDimension && hasVideo(p)) {
-        subscriptions.push({
-          userId: p.userId,
-          sessionId: p.sessionId,
-          trackType: TrackType.VIDEO,
-          dimension: p.videoDimension,
-        });
-      }
-      if (p.screenShareDimension && hasScreenShare(p)) {
-        subscriptions.push({
-          userId: p.userId,
-          sessionId: p.sessionId,
-          trackType: TrackType.SCREEN_SHARE,
-          dimension: p.screenShareDimension,
-        });
-      }
-      if (hasScreenShareAudio(p)) {
-        subscriptions.push({
-          userId: p.userId,
-          sessionId: p.sessionId,
-          trackType: TrackType.SCREEN_SHARE_AUDIO,
-        });
-      }
-    }
-    // schedule update
-    this.trackSubscriptionsSubject.next({ type, data: subscriptions });
-  };
-
   /**
    * Will enhance the reported stats with additional participant-specific information (`callStatsReport$` state [store variable](./StreamVideoClient.md/#readonlystatestore)).
    * This is usually helpful when detailed stats for a specific participant are needed.
@@ -2387,4 +2274,41 @@ export class Call {
       imageElement.removeEventListener('error', handleError);
     };
   };
+
+  /**
+   * Specify preference for incoming video resolution. The preference will
+   * be matched as close as possible, but actual resolution will depend
+   * on the video source quality and client network conditions. Will enable
+   * incoming video, if previously disabled.
+   *
+   * @param resolution preferred resolution, or `undefined` to clear preference
+   * @param sessionIds optionally specify session ids of the participants this
+   * preference has effect on. Affects all participants by default.
+   */
+  setPreferredIncomingVideoResolution = (
+    resolution: VideoResolution | undefined,
+    sessionIds?: string[],
+  ) => {
+    this.dynascaleManager.setVideoTrackSubscriptionOverrides(
+      resolution
+        ? {
+            enabled: true,
+            dimension: resolution,
+          }
+        : undefined,
+      sessionIds,
+    );
+    this.dynascaleManager.applyTrackSubscriptions();
+  };
+
+  /**
+   * Enables or disables incoming video from all remote call participants,
+   * and removes any preference for preferred resolution.
+   */
+  setIncomingVideoEnabled = (enabled: boolean) => {
+    this.dynascaleManager.setVideoTrackSubscriptionOverrides(
+      enabled ? undefined : { enabled: false },
+    );
+    this.dynascaleManager.applyTrackSubscriptions();
+  };
 }