fix: overridable bitrate and bitrate downscale factor (#1493)

Adds experimental APIs for overriding codec, bitrate, and bitrate downscale factor. Currently, we keep this API internal and undocumented until we get more feedback and more insights from our testing. In Pronto, these parameters can be overridden via a query parameter: `?video_codec=h264&bitrate=1200000&bitrate_factor=3` - will use h264, with 1.2mbps bitrate, and every lower layer will use 3 times less bitrate than the previous one.
GetStream · Sep 20, 2024 · cce5d8e · cce5d8e
1 parent 7346ddb
commit cce5d8e
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 121 deletions.
diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts
@@ -4,9 +4,7 @@ import { CameraDirection, CameraManagerState } from './CameraManagerState';
 import { InputMediaDeviceManager } from './InputMediaDeviceManager';
 import { getVideoDevices, getVideoStream } from './devices';
 import { TrackType } from '../gen/video/sfu/models/models';
-import { PublishOptions } from '../types';
-
-type PreferredCodec = 'vp8' | 'h264' | string;
+import { PreferredCodec, PublishOptions } from '../types';
 
 export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
   private targetResolution = {
@@ -15,35 +13,21 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
   };
 
   /**
-   * The preferred codec for encoding the video.
+   * The options to use when publishing the video stream.
    *
-   * @internal internal use only, not part of the public API.
+   * @internal
    */
-  preferredCodec: PreferredCodec | undefined;
+  publishOptions: PublishOptions | undefined;
 
   /**
-   * The preferred bitrate for encoding the video.
+   * Constructs a new CameraManager.
    *
-   * @internal internal use only, not part of the public API.
+   * @param call the call instance.
    */
-  preferredBitrate: number | undefined;
-
   constructor(call: Call) {
     super(call, new CameraManagerState(), TrackType.VIDEO);
   }
 
-  /**
-   * The publish options for the camera.
-   *
-   * @internal internal use only, not part of the public API.
-   */
-  get publishOptions(): PublishOptions {
-    return {
-      preferredCodec: this.preferredCodec,
-      preferredBitrate: this.preferredBitrate,
-    };
-  }
-
   /**
    * Select the camera direction.
    *
@@ -104,18 +88,36 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
    * @internal internal use only, not part of the public API.
    * @param codec the codec to use for encoding the video.
    */
-  setPreferredCodec(codec: 'vp8' | 'h264' | string | undefined) {
-    this.preferredCodec = codec;
+  setPreferredCodec(codec: PreferredCodec | undefined) {
+    this.updatePublishOptions({ preferredCodec: codec });
   }
 
   /**
-   * Sets the preferred bitrate for encoding the video.
+   * Updates the preferred publish options for the video stream.
    *
-   * @internal internal use only, not part of the public API.
-   * @param bitrate the bitrate to use for encoding the video.
+   * @internal
+   * @param options the options to use.
    */
-  setPreferredBitrate(bitrate: number | undefined) {
-    this.preferredBitrate = bitrate;
+  updatePublishOptions(options: PublishOptions) {
+    this.publishOptions = { ...this.publishOptions, ...options };
+  }
+
+  /**
+   * Returns the capture resolution of the camera.
+   */
+  getCaptureResolution() {
+    const { mediaStream } = this.state;
+    if (!mediaStream) return;
+
+    const [videoTrack] = mediaStream.getVideoTracks();
+    if (!videoTrack) return;
+
+    const settings = videoTrack.getSettings();
+    return {
+      width: settings.width,
+      height: settings.height,
+      frameRate: settings.frameRate,
+    };
   }
 
   protected getDevices(): Observable<MediaDeviceInfo[]> {

diff --git a/packages/client/src/devices/__tests__/CameraManager.test.ts b/packages/client/src/devices/__tests__/CameraManager.test.ts
@@ -82,9 +82,7 @@ describe('CameraManager', () => {
 
     expect(manager['call'].publishVideoStream).toHaveBeenCalledWith(
       manager.state.mediaStream,
-      {
-        preferredCodec: undefined,
-      },
+      undefined,
     );
   });
 

diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts
@@ -259,16 +259,11 @@ export class Publisher {
       const screenShareBitrate =
         settings?.screensharing.target_resolution?.bitrate;
 
-      const { preferredBitrate, preferredCodec, screenShareSettings } = opts;
       const videoEncodings =
         trackType === TrackType.VIDEO
-          ? findOptimalVideoLayers(track, targetResolution, preferredBitrate)
+          ? findOptimalVideoLayers(track, targetResolution, opts)
           : trackType === TrackType.SCREEN_SHARE
-            ? findOptimalScreenSharingLayers(
-                track,
-                screenShareSettings,
-                screenShareBitrate,
-              )
+            ? findOptimalScreenSharingLayers(track, opts, screenShareBitrate)
             : undefined;
 
       // listen for 'ended' event on the track as it might be ended abruptly
@@ -293,6 +288,7 @@ export class Publisher {
       this.transceiverRegistry[trackType] = transceiver;
       this.publishOptionsPerTrackType.set(trackType, opts);
 
+      const { preferredCodec } = opts;
       const codec =
         isReactNative() && trackType === TrackType.VIDEO && !preferredCodec
           ? getRNOptimalCodec()
@@ -713,16 +709,9 @@ export class Publisher {
           const publishOpts = this.publishOptionsPerTrackType.get(trackType);
           optimalLayers =
             trackType === TrackType.VIDEO
-              ? findOptimalVideoLayers(
-                  track,
-                  targetResolution,
-                  publishOpts?.preferredBitrate,
-                )
+              ? findOptimalVideoLayers(track, targetResolution, publishOpts)
               : trackType === TrackType.SCREEN_SHARE
-                ? findOptimalScreenSharingLayers(
-                    track,
-                    publishOpts?.screenShareSettings,
-                  )
+                ? findOptimalScreenSharingLayers(track, publishOpts)
                 : [];
           this.trackLayersCache[trackType] = optimalLayers;
         } else {

diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts
@@ -1,4 +1,4 @@
-import { ScreenShareSettings } from '../types';
+import { PublishOptions } from '../types';
 import { TargetResolutionResponse } from '../gen/shims';
 
 export type OptimalVideoLayer = RTCRtpEncodingParameters & {
@@ -25,24 +25,25 @@ const defaultBitratePerRid: Record<string, number> = {
  *
  * @param videoTrack the video track to find optimal layers for.
  * @param targetResolution the expected target resolution.
- * @param preferredBitrate the preferred bitrate for the video track.
+ * @param publishOptions the publish options for the track.
  */
 export const findOptimalVideoLayers = (
   videoTrack: MediaStreamTrack,
   targetResolution: TargetResolutionResponse = defaultTargetResolution,
-  preferredBitrate: number | undefined,
+  publishOptions?: PublishOptions,
 ) => {
   const optimalVideoLayers: OptimalVideoLayer[] = [];
   const settings = videoTrack.getSettings();
   const { width: w = 0, height: h = 0 } = settings;
-
+  const { preferredBitrate, bitrateDownscaleFactor = 2 } = publishOptions || {};
   const maxBitrate = getComputedMaxBitrate(
     targetResolution,
     w,
     h,
     preferredBitrate,
   );
   let downscaleFactor = 1;
+  let bitrateFactor = 1;
   ['f', 'h', 'q'].forEach((rid) => {
     // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index
     // when deciding which layer to disable when CPU or bandwidth is constrained.
@@ -53,11 +54,12 @@ export const findOptimalVideoLayers = (
       width: Math.round(w / downscaleFactor),
       height: Math.round(h / downscaleFactor),
       maxBitrate:
-        Math.round(maxBitrate / downscaleFactor) || defaultBitratePerRid[rid],
+        Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid],
       scaleResolutionDownBy: downscaleFactor,
       maxFramerate: 30,
     });
     downscaleFactor *= 2;
+    bitrateFactor *= bitrateDownscaleFactor;
   });
 
   // for simplicity, we start with all layers enabled, then this function
@@ -135,9 +137,10 @@ const withSimulcastConstraints = (
 
 export const findOptimalScreenSharingLayers = (
   videoTrack: MediaStreamTrack,
-  preferences?: ScreenShareSettings,
+  publishOptions?: PublishOptions,
   defaultMaxBitrate = 3000000,
 ): OptimalVideoLayer[] => {
+  const { screenShareSettings: preferences } = publishOptions || {};
   const settings = videoTrack.getSettings();
   return [
     {

diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts
@@ -146,9 +146,16 @@ export type SubscriptionChanges = {
   [sessionId: string]: SubscriptionChange;
 };
 
+/**
+ * A preferred codec to use when publishing a video track.
+ * @internal
+ */
+export type PreferredCodec = 'vp8' | 'h264' | string;
+
 export type PublishOptions = {
-  preferredCodec?: string | null;
+  preferredCodec?: PreferredCodec | null;
   preferredBitrate?: number;
+  bitrateDownscaleFactor?: number;
   screenShareSettings?: ScreenShareSettings;
 };
 

diff --git a/sample-apps/react/react-dogfood/components/DevMenu.tsx b/sample-apps/react/react-dogfood/components/DevMenu.tsx
@@ -1,8 +1,8 @@
-import Link from 'next/link';
 import { Icon, useCall, useCallStateHooks } from '@stream-io/video-react-sdk';
 import { decodeBase64 } from 'stream-chat';
 
 export const DevMenu = () => {
+  const call = useCall();
   return (
     <ul className="rd__dev-menu">
       <li className="rd__dev-menu__item">
@@ -57,16 +57,32 @@ export const DevMenu = () => {
         </a>
       </li>
       <li className="rd__dev-menu__item">
-        <Link
+        <a
           className="rd__link rd__link--faux-button rd__link--align-left"
           href={`${process.env.NEXT_PUBLIC_BASE_PATH || ''}/call-recordings`}
           target="_blank"
           rel="noreferrer"
         >
           <Icon className="rd__link__icon" icon="film-roll" />
           Recordings
-        </Link>
+        </a>
       </li>
+
+      <li className="rd__dev-menu__item rd__dev-menu__item--divider" />
+      <a
+        className="rd__link rd__link--faux-button rd__link--align-left"
+        href={`https://pronto-staging.getstream.io/join/${call?.id}?type=${call?.type}`}
+        rel="noreferrer"
+      >
+        Switch to Pronto Staging
+      </a>
+      <a
+        className="rd__link rd__link--faux-button rd__link--align-left"
+        href={`https://pronto.getstream.io/join/${call?.id}?type=${call?.type}`}
+        rel="noreferrer"
+      >
+        Switch to Pronto
+      </a>
     </ul>
   );
 };
@@ -139,61 +155,6 @@ const GoOrStopLive = () => {
   );
 };
 
-// const MigrateToNewSfu = () => {
-//   const call = useCall();
-//   return (
-//     <button className="rd__button rd__button--align-left"
-//       hidden
-//       onClick={() => {
-//         if (!call) return;
-//         call['dispatcher'].dispatch({
-//           eventPayload: {
-//             oneofKind: 'goAway',
-//             goAway: {
-//               reason: SfuModels.GoAwayReason.REBALANCE,
-//             },
-//           },
-//         });
-//       }}
-//     >
-//       <ListItemIcon>
-//         <SwitchAccessShortcutIcon
-//           fontSize="small"
-//           sx={{
-//             transform: 'rotate(90deg)',
-//           }}
-//         />
-//       </ListItemIcon>
-//       <ListItemText>Migrate to a new SFU</ListItemText>
-//     </button>
-//   );
-// };
-//
-
-// const EmulateSFUShuttingDown = () => {
-//   const call = useCall();
-//   return (
-//     <button className="rd__button rd__button--align-left"
-//       onClick={() => {
-//         if (!call) return;
-//         call['dispatcher'].dispatch({
-//           eventPayload: {
-//             oneofKind: 'error',
-//             error: {
-//               code: SfuModels.ErrorCode.SFU_SHUTTING_DOWN,
-//             },
-//           },
-//         });
-//       }}
-//     >
-//       <ListItemIcon>
-//         <PowerSettingsNewIcon fontSize="small" />
-//       </ListItemIcon>
-//       <ListItemText>Emulate SFU shutdown</ListItemText>
-//     </button>
-//   );
-// };
-
 const ConnectToLocalSfu = (props: { port?: number; sfuId?: string }) => {
   const { port = 3031, sfuId = 'SFU-1' } = props;
   const params = new URLSearchParams(window.location.search);

diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx
@@ -49,6 +49,10 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => {
 
   const isProntoEnvironment = useIsProntoEnvironment();
   const videoCodecOverride = router.query['video_codec'] as string | undefined;
+  const bitrateOverride = router.query['bitrate'] as string | undefined;
+  const bitrateFactorOverride = router.query['bitrate_factor'] as
+    | string
+    | undefined;
 
   const onJoin = useCallback(
     async ({ fastJoin = false } = {}) => {
@@ -59,15 +63,22 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => {
         const preferredCodec = videoCodecOverride || prontoDefaultCodec;
 
         const videoSettings = call.state.settings?.video;
-        const frameHeight = videoSettings?.target_resolution.height || 1080;
+        const frameHeight =
+          call.camera.getCaptureResolution()?.height ??
+          videoSettings?.target_resolution.height ??
+          1080;
 
-        const preferredBitrate = getPreferredBitrate(
-          preferredCodec,
-          frameHeight,
-        );
+        const preferredBitrate = bitrateOverride
+          ? parseInt(bitrateOverride, 10)
+          : getPreferredBitrate(preferredCodec, frameHeight);
 
-        call.camera.setPreferredBitrate(preferredBitrate);
-        call.camera.setPreferredCodec(preferredCodec);
+        call.camera.updatePublishOptions({
+          preferredCodec,
+          preferredBitrate,
+          bitrateDownscaleFactor: bitrateFactorOverride
+            ? parseInt(bitrateFactorOverride, 10)
+            : 2, // default to 2
+        });
 
         await call.join({ create: true });
         setShow('active-call');
@@ -77,7 +88,13 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => {
         setShow('error-join');
       }
     },
-    [call, isProntoEnvironment, videoCodecOverride],
+    [
+      bitrateFactorOverride,
+      bitrateOverride,
+      call,
+      isProntoEnvironment,
+      videoCodecOverride,
+    ],
   );
 
   const onLeave = useCallback(