From cce5d8e641a9182a1779952e4e62aa16ec21ab92 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 20 Sep 2024 17:21:26 +0200 Subject: [PATCH] fix: overridable bitrate and bitrate downscale factor (#1493) Adds experimental APIs for overriding codec, bitrate, and bitrate downscale factor. Currently, we keep this API internal and undocumented until we get more feedback and more insights from our testing. In Pronto, these parameters can be overridden via a query parameter: `?video_codec=h264&bitrate=1200000&bitrate_factor=3` - will use h264, with 1.2mbps bitrate, and every lower layer will use 3 times less bitrate than the previous one. --- packages/client/src/devices/CameraManager.ts | 60 ++++++++------- .../devices/__tests__/CameraManager.test.ts | 4 +- packages/client/src/rtc/Publisher.ts | 21 ++--- packages/client/src/rtc/videoLayers.ts | 15 ++-- packages/client/src/types.ts | 9 ++- .../react-dogfood/components/DevMenu.tsx | 77 +++++-------------- .../react-dogfood/components/MeetingUI.tsx | 33 ++++++-- 7 files changed, 98 insertions(+), 121 deletions(-) diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index ffb12e4d6f..f82aef4eb7 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -4,9 +4,7 @@ import { CameraDirection, CameraManagerState } from './CameraManagerState'; import { InputMediaDeviceManager } from './InputMediaDeviceManager'; import { getVideoDevices, getVideoStream } from './devices'; import { TrackType } from '../gen/video/sfu/models/models'; -import { PublishOptions } from '../types'; - -type PreferredCodec = 'vp8' | 'h264' | string; +import { PreferredCodec, PublishOptions } from '../types'; export class CameraManager extends InputMediaDeviceManager { private targetResolution = { @@ -15,35 +13,21 @@ export class CameraManager extends InputMediaDeviceManager { }; /** - * The preferred codec for encoding the video. + * The options to use when publishing the video stream. * - * @internal internal use only, not part of the public API. + * @internal */ - preferredCodec: PreferredCodec | undefined; + publishOptions: PublishOptions | undefined; /** - * The preferred bitrate for encoding the video. + * Constructs a new CameraManager. * - * @internal internal use only, not part of the public API. + * @param call the call instance. */ - preferredBitrate: number | undefined; - constructor(call: Call) { super(call, new CameraManagerState(), TrackType.VIDEO); } - /** - * The publish options for the camera. - * - * @internal internal use only, not part of the public API. - */ - get publishOptions(): PublishOptions { - return { - preferredCodec: this.preferredCodec, - preferredBitrate: this.preferredBitrate, - }; - } - /** * Select the camera direction. * @@ -104,18 +88,36 @@ export class CameraManager extends InputMediaDeviceManager { * @internal internal use only, not part of the public API. * @param codec the codec to use for encoding the video. */ - setPreferredCodec(codec: 'vp8' | 'h264' | string | undefined) { - this.preferredCodec = codec; + setPreferredCodec(codec: PreferredCodec | undefined) { + this.updatePublishOptions({ preferredCodec: codec }); } /** - * Sets the preferred bitrate for encoding the video. + * Updates the preferred publish options for the video stream. * - * @internal internal use only, not part of the public API. - * @param bitrate the bitrate to use for encoding the video. + * @internal + * @param options the options to use. */ - setPreferredBitrate(bitrate: number | undefined) { - this.preferredBitrate = bitrate; + updatePublishOptions(options: PublishOptions) { + this.publishOptions = { ...this.publishOptions, ...options }; + } + + /** + * Returns the capture resolution of the camera. + */ + getCaptureResolution() { + const { mediaStream } = this.state; + if (!mediaStream) return; + + const [videoTrack] = mediaStream.getVideoTracks(); + if (!videoTrack) return; + + const settings = videoTrack.getSettings(); + return { + width: settings.width, + height: settings.height, + frameRate: settings.frameRate, + }; } protected getDevices(): Observable { diff --git a/packages/client/src/devices/__tests__/CameraManager.test.ts b/packages/client/src/devices/__tests__/CameraManager.test.ts index 8d8fb8d0e4..656b5e48b2 100644 --- a/packages/client/src/devices/__tests__/CameraManager.test.ts +++ b/packages/client/src/devices/__tests__/CameraManager.test.ts @@ -82,9 +82,7 @@ describe('CameraManager', () => { expect(manager['call'].publishVideoStream).toHaveBeenCalledWith( manager.state.mediaStream, - { - preferredCodec: undefined, - }, + undefined, ); }); diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index f35d9dd6d4..8ef17b23a8 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -259,16 +259,11 @@ export class Publisher { const screenShareBitrate = settings?.screensharing.target_resolution?.bitrate; - const { preferredBitrate, preferredCodec, screenShareSettings } = opts; const videoEncodings = trackType === TrackType.VIDEO - ? findOptimalVideoLayers(track, targetResolution, preferredBitrate) + ? findOptimalVideoLayers(track, targetResolution, opts) : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers( - track, - screenShareSettings, - screenShareBitrate, - ) + ? findOptimalScreenSharingLayers(track, opts, screenShareBitrate) : undefined; // listen for 'ended' event on the track as it might be ended abruptly @@ -293,6 +288,7 @@ export class Publisher { this.transceiverRegistry[trackType] = transceiver; this.publishOptionsPerTrackType.set(trackType, opts); + const { preferredCodec } = opts; const codec = isReactNative() && trackType === TrackType.VIDEO && !preferredCodec ? getRNOptimalCodec() @@ -713,16 +709,9 @@ export class Publisher { const publishOpts = this.publishOptionsPerTrackType.get(trackType); optimalLayers = trackType === TrackType.VIDEO - ? findOptimalVideoLayers( - track, - targetResolution, - publishOpts?.preferredBitrate, - ) + ? findOptimalVideoLayers(track, targetResolution, publishOpts) : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers( - track, - publishOpts?.screenShareSettings, - ) + ? findOptimalScreenSharingLayers(track, publishOpts) : []; this.trackLayersCache[trackType] = optimalLayers; } else { diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index ce29bed9da..649d98e988 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,4 +1,4 @@ -import { ScreenShareSettings } from '../types'; +import { PublishOptions } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { @@ -25,17 +25,17 @@ const defaultBitratePerRid: Record = { * * @param videoTrack the video track to find optimal layers for. * @param targetResolution the expected target resolution. - * @param preferredBitrate the preferred bitrate for the video track. + * @param publishOptions the publish options for the track. */ export const findOptimalVideoLayers = ( videoTrack: MediaStreamTrack, targetResolution: TargetResolutionResponse = defaultTargetResolution, - preferredBitrate: number | undefined, + publishOptions?: PublishOptions, ) => { const optimalVideoLayers: OptimalVideoLayer[] = []; const settings = videoTrack.getSettings(); const { width: w = 0, height: h = 0 } = settings; - + const { preferredBitrate, bitrateDownscaleFactor = 2 } = publishOptions || {}; const maxBitrate = getComputedMaxBitrate( targetResolution, w, @@ -43,6 +43,7 @@ export const findOptimalVideoLayers = ( preferredBitrate, ); let downscaleFactor = 1; + let bitrateFactor = 1; ['f', 'h', 'q'].forEach((rid) => { // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index // when deciding which layer to disable when CPU or bandwidth is constrained. @@ -53,11 +54,12 @@ export const findOptimalVideoLayers = ( width: Math.round(w / downscaleFactor), height: Math.round(h / downscaleFactor), maxBitrate: - Math.round(maxBitrate / downscaleFactor) || defaultBitratePerRid[rid], + Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid], scaleResolutionDownBy: downscaleFactor, maxFramerate: 30, }); downscaleFactor *= 2; + bitrateFactor *= bitrateDownscaleFactor; }); // for simplicity, we start with all layers enabled, then this function @@ -135,9 +137,10 @@ const withSimulcastConstraints = ( export const findOptimalScreenSharingLayers = ( videoTrack: MediaStreamTrack, - preferences?: ScreenShareSettings, + publishOptions?: PublishOptions, defaultMaxBitrate = 3000000, ): OptimalVideoLayer[] => { + const { screenShareSettings: preferences } = publishOptions || {}; const settings = videoTrack.getSettings(); return [ { diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 14ff4a7968..f6c83934b6 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -146,9 +146,16 @@ export type SubscriptionChanges = { [sessionId: string]: SubscriptionChange; }; +/** + * A preferred codec to use when publishing a video track. + * @internal + */ +export type PreferredCodec = 'vp8' | 'h264' | string; + export type PublishOptions = { - preferredCodec?: string | null; + preferredCodec?: PreferredCodec | null; preferredBitrate?: number; + bitrateDownscaleFactor?: number; screenShareSettings?: ScreenShareSettings; }; diff --git a/sample-apps/react/react-dogfood/components/DevMenu.tsx b/sample-apps/react/react-dogfood/components/DevMenu.tsx index 0ada994824..b5f143136b 100644 --- a/sample-apps/react/react-dogfood/components/DevMenu.tsx +++ b/sample-apps/react/react-dogfood/components/DevMenu.tsx @@ -1,8 +1,8 @@ -import Link from 'next/link'; import { Icon, useCall, useCallStateHooks } from '@stream-io/video-react-sdk'; import { decodeBase64 } from 'stream-chat'; export const DevMenu = () => { + const call = useCall(); return ( ); }; @@ -139,61 +155,6 @@ const GoOrStopLive = () => { ); }; -// const MigrateToNewSfu = () => { -// const call = useCall(); -// return ( -// -// ); -// }; -// - -// const EmulateSFUShuttingDown = () => { -// const call = useCall(); -// return ( -// -// ); -// }; - const ConnectToLocalSfu = (props: { port?: number; sfuId?: string }) => { const { port = 3031, sfuId = 'SFU-1' } = props; const params = new URLSearchParams(window.location.search); diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 9fdd2ee5c4..0401b50988 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -49,6 +49,10 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const isProntoEnvironment = useIsProntoEnvironment(); const videoCodecOverride = router.query['video_codec'] as string | undefined; + const bitrateOverride = router.query['bitrate'] as string | undefined; + const bitrateFactorOverride = router.query['bitrate_factor'] as + | string + | undefined; const onJoin = useCallback( async ({ fastJoin = false } = {}) => { @@ -59,15 +63,22 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const preferredCodec = videoCodecOverride || prontoDefaultCodec; const videoSettings = call.state.settings?.video; - const frameHeight = videoSettings?.target_resolution.height || 1080; + const frameHeight = + call.camera.getCaptureResolution()?.height ?? + videoSettings?.target_resolution.height ?? + 1080; - const preferredBitrate = getPreferredBitrate( - preferredCodec, - frameHeight, - ); + const preferredBitrate = bitrateOverride + ? parseInt(bitrateOverride, 10) + : getPreferredBitrate(preferredCodec, frameHeight); - call.camera.setPreferredBitrate(preferredBitrate); - call.camera.setPreferredCodec(preferredCodec); + call.camera.updatePublishOptions({ + preferredCodec, + preferredBitrate, + bitrateDownscaleFactor: bitrateFactorOverride + ? parseInt(bitrateFactorOverride, 10) + : 2, // default to 2 + }); await call.join({ create: true }); setShow('active-call'); @@ -77,7 +88,13 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { setShow('error-join'); } }, - [call, isProntoEnvironment, videoCodecOverride], + [ + bitrateFactorOverride, + bitrateOverride, + call, + isProntoEnvironment, + videoCodecOverride, + ], ); const onLeave = useCallback(