From df1f24058ec170988b5eae4a47d369db4e5f0858 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 4 Jul 2024 18:31:21 +0200 Subject: [PATCH 01/33] feat(vp9): experimental VP9 support --- packages/client/src/devices/CameraManager.ts | 12 ++++++++++-- packages/client/src/rtc/Publisher.ts | 4 ++-- packages/client/src/rtc/videoLayers.ts | 10 ++++++++-- .../react/react-dogfood/components/MeetingUI.tsx | 5 ++++- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index da238776e8..88519aa765 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -5,7 +5,7 @@ import { InputMediaDeviceManager } from './InputMediaDeviceManager'; import { getVideoDevices, getVideoStream } from './devices'; import { TrackType } from '../gen/video/sfu/models/models'; -type PreferredCodec = 'vp8' | 'h264' | string; +type PreferredCodec = 'vp8' | 'vp9' | 'h264' | string; export class CameraManager extends InputMediaDeviceManager { private targetResolution = { @@ -19,6 +19,11 @@ export class CameraManager extends InputMediaDeviceManager { * @internal internal use only, not part of the public API. */ preferredCodec: PreferredCodec | undefined; + /** + * The scalability mode to use for the codec. + * @internal internal use only, not part of the public API. + */ + scalabilityMode: string | undefined; constructor(call: Call) { super(call, new CameraManagerState(), TrackType.VIDEO); @@ -83,9 +88,11 @@ export class CameraManager extends InputMediaDeviceManager { * * @internal internal use only, not part of the public API. * @param codec the codec to use for encoding the video. + * @param scalabilityMode the scalability mode to use for the codec. */ - setPreferredCodec(codec: 'vp8' | 'h264' | string | undefined) { + setPreferredCodec(codec: PreferredCodec, scalabilityMode?: string) { this.preferredCodec = codec; + this.scalabilityMode = scalabilityMode; } protected getDevices(): Observable { @@ -109,6 +116,7 @@ export class CameraManager extends InputMediaDeviceManager { protected publishStream(stream: MediaStream): Promise { return this.call.publishVideoStream(stream, { preferredCodec: this.preferredCodec, + scalabilityMode: this.scalabilityMode, }); } diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 7cdb12b863..4911b44466 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -252,7 +252,7 @@ export class Publisher { const videoEncodings = trackType === TrackType.VIDEO - ? findOptimalVideoLayers(track, targetResolution) + ? findOptimalVideoLayers(track, targetResolution, opts) : trackType === TrackType.SCREEN_SHARE ? findOptimalScreenSharingLayers( track, @@ -751,7 +751,7 @@ export class Publisher { const publishOpts = this.publishOptionsPerTrackType.get(trackType); optimalLayers = trackType === TrackType.VIDEO - ? findOptimalVideoLayers(track, targetResolution) + ? findOptimalVideoLayers(track, targetResolution, undefined) : trackType === TrackType.SCREEN_SHARE ? findOptimalScreenSharingLayers( track, diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 52b91017e9..1fa0b2ae51 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,5 +1,5 @@ import { getOSInfo } from '../client-details'; -import { ScreenShareSettings } from '../types'; +import { PublishOptions, ScreenShareSettings } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; import { isReactNative } from '../helpers/platforms'; @@ -27,10 +27,12 @@ const defaultBitratePerRid: Record = { * * @param videoTrack the video track to find optimal layers for. * @param targetResolution the expected target resolution. + * @param options the publish options. */ export const findOptimalVideoLayers = ( videoTrack: MediaStreamTrack, targetResolution: TargetResolutionResponse = defaultTargetResolution, + options: PublishOptions = {}, ) => { const optimalVideoLayers: OptimalVideoLayer[] = []; const settings = videoTrack.getSettings(); @@ -40,7 +42,8 @@ export const findOptimalVideoLayers = ( const maxBitrate = getComputedMaxBitrate(targetResolution, w, h); let downscaleFactor = 1; - ['f', 'h', 'q'].forEach((rid) => { + const { preferredCodec, scalabilityMode } = options; + (preferredCodec === 'vp9' ? ['q'] : ['f', 'h', 'q']).forEach((rid) => { // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index // when deciding which layer to disable when CPU or bandwidth is constrained. // Encodings should be ordered in increasing spatial resolution order. @@ -58,6 +61,9 @@ export const findOptimalVideoLayers = ( h: isRNIos ? 30 : 25, q: isRNIos ? 30 : 20, }[rid], + ...(preferredCodec === 'vp9' + ? { scalabilityMode: scalabilityMode || 'L3T3' } + : null), }); downscaleFactor *= 2; }); diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 1efc5ca4af..4c043275f3 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -50,8 +50,11 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { if (!fastJoin) setShow('loading'); try { const preferredCodec = router.query['video_codec']; + const scalabilityMode = router.query['scalability_mode'] as + | string + | undefined; if (typeof preferredCodec === 'string') { - activeCall?.camera.setPreferredCodec(preferredCodec); + activeCall?.camera.setPreferredCodec(preferredCodec, scalabilityMode); } await activeCall?.join({ create: true }); setShow('active-call'); From 653406213ed3672adab44760f7d9cac57f52aa42 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 4 Jul 2024 18:34:37 +0200 Subject: [PATCH 02/33] fix: missing type --- packages/client/src/types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 0798478314..1950e6ae9c 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -147,7 +147,8 @@ export type SubscriptionChanges = { }; export type PublishOptions = { - preferredCodec?: string | null; + preferredCodec?: string; + scalabilityMode?: string; screenShareSettings?: ScreenShareSettings; }; From fe7f0c41490f3357d5eec41c00490eff103613ef Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 5 Jul 2024 09:43:09 +0200 Subject: [PATCH 03/33] fix: publish 30fps for all qualities --- packages/client/src/rtc/videoLayers.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 1fa0b2ae51..016f5ca35d 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,7 +1,5 @@ -import { getOSInfo } from '../client-details'; import { PublishOptions, ScreenShareSettings } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; -import { isReactNative } from '../helpers/platforms'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { width: number; @@ -38,8 +36,6 @@ export const findOptimalVideoLayers = ( const settings = videoTrack.getSettings(); const { width: w = 0, height: h = 0 } = settings; - const isRNIos = isReactNative() && getOSInfo()?.name.toLowerCase() === 'ios'; - const maxBitrate = getComputedMaxBitrate(targetResolution, w, h); let downscaleFactor = 1; const { preferredCodec, scalabilityMode } = options; @@ -55,12 +51,7 @@ export const findOptimalVideoLayers = ( maxBitrate: Math.round(maxBitrate / downscaleFactor) || defaultBitratePerRid[rid], scaleResolutionDownBy: downscaleFactor, - // Simulcast on iOS React-Native requires all encodings to share the same framerate - maxFramerate: { - f: 30, - h: isRNIos ? 30 : 25, - q: isRNIos ? 30 : 20, - }[rid], + maxFramerate: 30, ...(preferredCodec === 'vp9' ? { scalabilityMode: scalabilityMode || 'L3T3' } : null), From d45991fd1a1404b32f42b0f255c83be57a50df4a Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 5 Jul 2024 09:51:53 +0200 Subject: [PATCH 04/33] fix: publish 30fps for all qualities --- packages/client/src/rtc/__tests__/videoLayers.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index d32b632b8a..e2826b1e6d 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -59,7 +59,7 @@ describe('videoLayers', () => { height: height / 4, maxBitrate: targetBitrate / 4, scaleResolutionDownBy: 4, - maxFramerate: 20, + maxFramerate: 30, }, { active: true, @@ -68,7 +68,7 @@ describe('videoLayers', () => { height: height / 2, maxBitrate: targetBitrate / 2, scaleResolutionDownBy: 2, - maxFramerate: 25, + maxFramerate: 30, }, { active: true, From 2d9d812f80b1f935bac8d3bc818f4765d884cac7 Mon Sep 17 00:00:00 2001 From: "Suchith.J.N" Date: Tue, 23 Jul 2024 17:13:21 +0200 Subject: [PATCH 05/33] enable av1 as well --- packages/client/src/devices/CameraManager.ts | 2 +- packages/client/src/rtc/videoLayers.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index 88519aa765..094ae654d1 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -5,7 +5,7 @@ import { InputMediaDeviceManager } from './InputMediaDeviceManager'; import { getVideoDevices, getVideoStream } from './devices'; import { TrackType } from '../gen/video/sfu/models/models'; -type PreferredCodec = 'vp8' | 'vp9' | 'h264' | string; +type PreferredCodec = 'vp8' | 'vp9' | 'av1' | 'h264' | string; export class CameraManager extends InputMediaDeviceManager { private targetResolution = { diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 016f5ca35d..8558d9caa8 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -39,7 +39,7 @@ export const findOptimalVideoLayers = ( const maxBitrate = getComputedMaxBitrate(targetResolution, w, h); let downscaleFactor = 1; const { preferredCodec, scalabilityMode } = options; - (preferredCodec === 'vp9' ? ['q'] : ['f', 'h', 'q']).forEach((rid) => { + ((preferredCodec === 'vp9' || preferredCodec === 'av1') ? ['q'] : ['f', 'h', 'q']).forEach((rid) => { // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index // when deciding which layer to disable when CPU or bandwidth is constrained. // Encodings should be ordered in increasing spatial resolution order. @@ -52,7 +52,7 @@ export const findOptimalVideoLayers = ( Math.round(maxBitrate / downscaleFactor) || defaultBitratePerRid[rid], scaleResolutionDownBy: downscaleFactor, maxFramerate: 30, - ...(preferredCodec === 'vp9' + ...((preferredCodec === 'vp9' || preferredCodec === 'av1') ? { scalabilityMode: scalabilityMode || 'L3T3' } : null), }); From c9d292ed733cff37ed006154e4d6b68c59041abd Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 23 Jul 2024 17:50:04 +0200 Subject: [PATCH 06/33] fix: apply correct formatting --- packages/client/src/rtc/videoLayers.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 8558d9caa8..80c83014d6 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -39,7 +39,10 @@ export const findOptimalVideoLayers = ( const maxBitrate = getComputedMaxBitrate(targetResolution, w, h); let downscaleFactor = 1; const { preferredCodec, scalabilityMode } = options; - ((preferredCodec === 'vp9' || preferredCodec === 'av1') ? ['q'] : ['f', 'h', 'q']).forEach((rid) => { + (preferredCodec === 'vp9' || preferredCodec === 'av1' + ? ['q'] + : ['f', 'h', 'q'] + ).forEach((rid) => { // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index // when deciding which layer to disable when CPU or bandwidth is constrained. // Encodings should be ordered in increasing spatial resolution order. @@ -52,7 +55,7 @@ export const findOptimalVideoLayers = ( Math.round(maxBitrate / downscaleFactor) || defaultBitratePerRid[rid], scaleResolutionDownBy: downscaleFactor, maxFramerate: 30, - ...((preferredCodec === 'vp9' || preferredCodec === 'av1') + ...(preferredCodec === 'vp9' || preferredCodec === 'av1' ? { scalabilityMode: scalabilityMode || 'L3T3' } : null), }); From a4c8710c2e8cc1743b84a4574cfe9759baf5e90d Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 1 Oct 2024 15:29:03 +0200 Subject: [PATCH 07/33] fix: improve video layer assignment --- packages/client/src/rtc/videoLayers.ts | 39 ++++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 445c2752c6..1a3394f5b3 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -4,6 +4,8 @@ import { TargetResolutionResponse } from '../gen/shims'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { width: number; height: number; + // NOTE OL: should be part of RTCRtpEncodingParameters + scalabilityMode?: string; }; const DEFAULT_BITRATE = 1250000; @@ -34,7 +36,7 @@ export const findOptimalVideoLayers = ( ) => { const optimalVideoLayers: OptimalVideoLayer[] = []; const settings = videoTrack.getSettings(); - const { width: w = 0, height: h = 0 } = settings; + const { width = 0, height = 0 } = settings; const { preferredCodec, scalabilityMode, @@ -43,35 +45,36 @@ export const findOptimalVideoLayers = ( } = publishOptions || {}; const maxBitrate = getComputedMaxBitrate( targetResolution, - w, - h, + width, + height, preferredBitrate, ); let downscaleFactor = 1; let bitrateFactor = 1; - (preferredCodec === 'vp9' || preferredCodec === 'av1' - ? ['q'] - : ['f', 'h', 'q'] - ).forEach((rid) => { - // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index - // when deciding which layer to disable when CPU or bandwidth is constrained. - // Encodings should be ordered in increasing spatial resolution order. - optimalVideoLayers.unshift({ + const isSvcCodec = preferredCodec === 'vp9' || preferredCodec === 'av1'; + const layers = isSvcCodec ? ['q'] : ['f', 'h', 'q']; + for (const rid of layers) { + const layer: OptimalVideoLayer = { active: true, rid, - width: Math.round(w / downscaleFactor), - height: Math.round(h / downscaleFactor), + width: Math.round(width / downscaleFactor), + height: Math.round(height / downscaleFactor), maxBitrate: Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid], scaleResolutionDownBy: downscaleFactor, maxFramerate: 30, - ...(preferredCodec === 'vp9' || preferredCodec === 'av1' - ? { scalabilityMode: scalabilityMode || 'L3T3' } - : null), - }); + }; + if (isSvcCodec) { + layer.scalabilityMode = scalabilityMode || 'L3T3_KEY'; + } + + // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index + // when deciding which layer to disable when CPU or bandwidth is constrained. + // Encodings should be ordered in increasing spatial resolution order. + optimalVideoLayers.unshift(layer); downscaleFactor *= 2; bitrateFactor *= bitrateDownscaleFactor; - }); + } // for simplicity, we start with all layers enabled, then this function // will clear/reassign the layers that are not needed From c336ae2e8edae700074be42da279b1cb65c5d384 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 1 Oct 2024 15:33:29 +0200 Subject: [PATCH 08/33] chore: add test --- .../client/src/rtc/__tests__/videoLayers.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index fccc65e7d1..59764b2c58 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -135,6 +135,21 @@ describe('videoLayers', () => { expect(layers[2].rid).toBe('f'); }); + it('should announce only one layer for SVC codecs', () => { + const track = new MediaStreamTrack(); + vi.spyOn(track, 'getSettings').mockReturnValue({ + width: 1280, + height: 720, + }); + const layers = findOptimalVideoLayers(track, undefined, { + preferredCodec: 'vp9', + scalabilityMode: 'L3T3', + }); + expect(layers.length).toBe(1); + expect(layers[0].rid).toBe('q'); + expect(layers[0].scalabilityMode).toBe('L3T3'); + }); + describe('getComputedMaxBitrate', () => { it('should scale target bitrate down if resolution is smaller than target resolution', () => { const targetResolution = { width: 1920, height: 1080, bitrate: 3000000 }; From d9b5f2e45249339dc6c8797baf047bb0dd273c8d Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 8 Oct 2024 17:45:17 +0200 Subject: [PATCH 09/33] feat: handle the `scalabilityMode` param in changePublishQuality --- packages/client/src/Call.ts | 15 +- .../client/src/events/callEventHandlers.ts | 2 - packages/client/src/events/internal.ts | 16 --- .../client/src/gen/video/sfu/event/events.ts | 128 ++---------------- packages/client/src/rtc/Publisher.ts | 127 +++++++++-------- .../react-dogfood/components/MeetingUI.tsx | 2 +- .../react-dogfood/helpers/bitrateLookup.ts | 16 ++- 7 files changed, 87 insertions(+), 219 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 31e5cf08ac..8a9a59e091 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -87,10 +87,7 @@ import { VideoTrackType, } from './types'; import { BehaviorSubject, Subject, takeWhile } from 'rxjs'; -import { - ReconnectDetails, - VideoLayerSetting, -} from './gen/video/sfu/event/events'; +import { ReconnectDetails } from './gen/video/sfu/event/events'; import { ClientDetails, TrackType, @@ -1529,16 +1526,6 @@ export class Call { return this.state.setSortParticipantsBy(criteria); }; - /** - * Updates the list of video layers to publish. - * - * @internal - * @param enabledLayers the list of layers to enable. - */ - updatePublishQuality = async (enabledLayers: VideoLayerSetting[]) => { - return this.publisher?.updateVideoPublishQuality(enabledLayers); - }; - /** * Sends a reaction to the other call participants. * diff --git a/packages/client/src/events/callEventHandlers.ts b/packages/client/src/events/callEventHandlers.ts index 642d48588f..e218baf91f 100644 --- a/packages/client/src/events/callEventHandlers.ts +++ b/packages/client/src/events/callEventHandlers.ts @@ -7,7 +7,6 @@ import { watchCallEnded, watchCallGrantsUpdated, watchCallRejected, - watchChangePublishQuality, watchConnectionQualityChanged, watchDominantSpeakerChanged, watchLiveEnded, @@ -46,7 +45,6 @@ export const registerEventHandlers = (call: Call, dispatcher: Dispatcher) => { watchLiveEnded(dispatcher, call), watchSfuErrorReports(dispatcher), - watchChangePublishQuality(dispatcher, call), watchConnectionQualityChanged(dispatcher, state), watchParticipantCountChanged(dispatcher, state), diff --git a/packages/client/src/events/internal.ts b/packages/client/src/events/internal.ts index b74719f74f..c5f08c35fd 100644 --- a/packages/client/src/events/internal.ts +++ b/packages/client/src/events/internal.ts @@ -10,22 +10,6 @@ import { } from '../gen/video/sfu/models/models'; import { OwnCapability } from '../gen/coordinator'; -/** - * An event responder which handles the `changePublishQuality` event. - */ -export const watchChangePublishQuality = ( - dispatcher: Dispatcher, - call: Call, -) => { - return dispatcher.on('changePublishQuality', (e) => { - const { videoSenders } = e; - videoSenders.forEach((videoSender) => { - const { layers } = videoSender; - call.updatePublishQuality(layers.filter((l) => l.active)); - }); - }); -}; - export const watchConnectionQualityChanged = ( dispatcher: Dispatcher, state: CallState, diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts index 48d5203110..4519a1e0b0 100644 --- a/packages/client/src/gen/video/sfu/event/events.ts +++ b/packages/client/src/gen/video/sfu/event/events.ts @@ -682,45 +682,15 @@ export interface AudioLevelChanged { */ audioLevels: AudioLevel[]; } -/** - * @generated from protobuf message stream.video.sfu.event.AudioMediaRequest - */ -export interface AudioMediaRequest { - /** - * @generated from protobuf field: int32 channel_count = 1; - */ - channelCount: number; -} /** * @generated from protobuf message stream.video.sfu.event.AudioSender */ export interface AudioSender { - /** - * @generated from protobuf field: stream.video.sfu.event.AudioMediaRequest media_request = 1; - */ - mediaRequest?: AudioMediaRequest; /** * @generated from protobuf field: stream.video.sfu.models.Codec codec = 2; */ codec?: Codec; } -/** - * @generated from protobuf message stream.video.sfu.event.VideoMediaRequest - */ -export interface VideoMediaRequest { - /** - * @generated from protobuf field: int32 ideal_height = 1; - */ - idealHeight: number; - /** - * @generated from protobuf field: int32 ideal_width = 2; - */ - idealWidth: number; - /** - * @generated from protobuf field: int32 ideal_frame_rate = 3; - */ - idealFrameRate: number; -} /** * VideoLayerSetting is used to specify various parameters of a particular encoding in simulcast. * The parameters are specified here - https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpEncodingParameters @@ -745,10 +715,6 @@ export interface VideoLayerSetting { * @generated from protobuf field: float scale_resolution_down_by = 4; */ scaleResolutionDownBy: number; - /** - * @generated from protobuf field: stream.video.sfu.event.VideoLayerSetting.Priority priority = 5; - */ - priority: VideoLayerSetting_Priority; /** * @generated from protobuf field: stream.video.sfu.models.Codec codec = 6; */ @@ -757,36 +723,15 @@ export interface VideoLayerSetting { * @generated from protobuf field: uint32 max_framerate = 7; */ maxFramerate: number; -} -/** - * @generated from protobuf enum stream.video.sfu.event.VideoLayerSetting.Priority - */ -export enum VideoLayerSetting_Priority { - /** - * @generated from protobuf enum value: PRIORITY_HIGH_UNSPECIFIED = 0; - */ - HIGH_UNSPECIFIED = 0, - /** - * @generated from protobuf enum value: PRIORITY_LOW = 1; - */ - LOW = 1, - /** - * @generated from protobuf enum value: PRIORITY_MEDIUM = 2; - */ - MEDIUM = 2, /** - * @generated from protobuf enum value: PRIORITY_VERY_LOW = 3; + * @generated from protobuf field: string scalability_mode = 8; */ - VERY_LOW = 3, + scalabilityMode: string; } /** * @generated from protobuf message stream.video.sfu.event.VideoSender */ export interface VideoSender { - /** - * @generated from protobuf field: stream.video.sfu.event.VideoMediaRequest media_request = 1; - */ - mediaRequest?: VideoMediaRequest; /** * @generated from protobuf field: stream.video.sfu.models.Codec codec = 2; */ @@ -1537,32 +1482,9 @@ class AudioLevelChanged$Type extends MessageType { */ export const AudioLevelChanged = new AudioLevelChanged$Type(); // @generated message type with reflection information, may provide speed optimized methods -class AudioMediaRequest$Type extends MessageType { - constructor() { - super('stream.video.sfu.event.AudioMediaRequest', [ - { - no: 1, - name: 'channel_count', - kind: 'scalar', - T: 5 /*ScalarType.INT32*/, - }, - ]); - } -} -/** - * @generated MessageType for protobuf message stream.video.sfu.event.AudioMediaRequest - */ -export const AudioMediaRequest = new AudioMediaRequest$Type(); -// @generated message type with reflection information, may provide speed optimized methods class AudioSender$Type extends MessageType { constructor() { super('stream.video.sfu.event.AudioSender', [ - { - no: 1, - name: 'media_request', - kind: 'message', - T: () => AudioMediaRequest, - }, { no: 2, name: 'codec', kind: 'message', T: () => Codec }, ]); } @@ -1572,30 +1494,6 @@ class AudioSender$Type extends MessageType { */ export const AudioSender = new AudioSender$Type(); // @generated message type with reflection information, may provide speed optimized methods -class VideoMediaRequest$Type extends MessageType { - constructor() { - super('stream.video.sfu.event.VideoMediaRequest', [ - { - no: 1, - name: 'ideal_height', - kind: 'scalar', - T: 5 /*ScalarType.INT32*/, - }, - { no: 2, name: 'ideal_width', kind: 'scalar', T: 5 /*ScalarType.INT32*/ }, - { - no: 3, - name: 'ideal_frame_rate', - kind: 'scalar', - T: 5 /*ScalarType.INT32*/, - }, - ]); - } -} -/** - * @generated MessageType for protobuf message stream.video.sfu.event.VideoMediaRequest - */ -export const VideoMediaRequest = new VideoMediaRequest$Type(); -// @generated message type with reflection information, may provide speed optimized methods class VideoLayerSetting$Type extends MessageType { constructor() { super('stream.video.sfu.event.VideoLayerSetting', [ @@ -1608,16 +1506,6 @@ class VideoLayerSetting$Type extends MessageType { kind: 'scalar', T: 2 /*ScalarType.FLOAT*/, }, - { - no: 5, - name: 'priority', - kind: 'enum', - T: () => [ - 'stream.video.sfu.event.VideoLayerSetting.Priority', - VideoLayerSetting_Priority, - 'PRIORITY_', - ], - }, { no: 6, name: 'codec', kind: 'message', T: () => Codec }, { no: 7, @@ -1625,6 +1513,12 @@ class VideoLayerSetting$Type extends MessageType { kind: 'scalar', T: 13 /*ScalarType.UINT32*/, }, + { + no: 8, + name: 'scalability_mode', + kind: 'scalar', + T: 9 /*ScalarType.STRING*/, + }, ]); } } @@ -1636,12 +1530,6 @@ export const VideoLayerSetting = new VideoLayerSetting$Type(); class VideoSender$Type extends MessageType { constructor() { super('stream.video.sfu.event.VideoSender', [ - { - no: 1, - name: 'media_request', - kind: 'message', - T: () => VideoMediaRequest, - }, { no: 2, name: 'codec', kind: 'message', T: () => Codec }, { no: 3, diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 8ef17b23a8..d7abf2682f 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -24,6 +24,7 @@ import { getLogger } from '../logger'; import { Dispatcher } from './Dispatcher'; import { VideoLayerSetting } from '../gen/video/sfu/event/events'; import { TargetResolutionResponse } from '../gen/shims'; +import { withoutConcurrency } from '../helpers/concurrency'; export type PublisherConstructorOpts = { sfuClient: StreamSfuClient; @@ -94,6 +95,7 @@ export class Publisher { private readonly isRedEnabled: boolean; private readonly unsubscribeOnIceRestart: () => void; + private readonly unsubscribeChangePublishQuality: () => void; private readonly onUnrecoverableError?: () => void; private isIceRestarting = false; @@ -141,6 +143,21 @@ export class Publisher { this.onUnrecoverableError?.(); }); }); + + this.unsubscribeChangePublishQuality = dispatcher.on( + 'changePublishQuality', + ({ videoSenders }) => { + withoutConcurrency('publisher.changePublishQuality', async () => { + for (const videoSender of videoSenders) { + const { layers } = videoSender; + const enabledLayers = layers.filter((l) => l.active); + await this.changePublishQuality(enabledLayers); + } + }).catch((err) => { + this.logger('warn', 'Failed to change publish quality', err); + }); + }, + ); } private createPeerConnection = (connectionConfig?: RTCConfiguration) => { @@ -188,6 +205,7 @@ export class Publisher { */ detachEventHandlers = () => { this.unsubscribeOnIceRestart(); + this.unsubscribeChangePublishQuality(); this.pc.removeEventListener('icecandidate', this.onIceCandidate); this.pc.removeEventListener('negotiationneeded', this.onNegotiationNeeded); @@ -406,7 +424,7 @@ export class Publisher { }); }; - updateVideoPublishQuality = async (enabledLayers: VideoLayerSetting[]) => { + private changePublishQuality = async (enabledLayers: VideoLayerSetting[]) => { this.logger( 'info', 'Update publish quality, requested layers by SFU:', @@ -429,78 +447,57 @@ export class Publisher { } let changed = false; - let enabledRids = enabledLayers - .filter((ly) => ly.active) - .map((ly) => ly.name); - params.encodings.forEach((enc) => { + for (const encoder of params.encodings) { + const layer = enabledLayers.find((vls) => vls.name === encoder.rid); + if (!layer) continue; + // flip 'active' flag only when necessary - const shouldEnable = enabledRids.includes(enc.rid!); - if (shouldEnable !== enc.active) { - enc.active = shouldEnable; + const shouldActivate = layer.active; + if (shouldActivate !== encoder.active) { + encoder.active = shouldActivate; changed = true; } - if (shouldEnable) { - let layer = enabledLayers.find((vls) => vls.name === enc.rid); - if (layer !== undefined) { - if ( - layer.scaleResolutionDownBy >= 1 && - layer.scaleResolutionDownBy !== enc.scaleResolutionDownBy - ) { - this.logger( - 'debug', - '[dynascale]: setting scaleResolutionDownBy from server', - 'layer', - layer.name, - 'scale-resolution-down-by', - layer.scaleResolutionDownBy, - ); - enc.scaleResolutionDownBy = layer.scaleResolutionDownBy; - changed = true; - } - if (layer.maxBitrate > 0 && layer.maxBitrate !== enc.maxBitrate) { - this.logger( - 'debug', - '[dynascale] setting max-bitrate from the server', - 'layer', - layer.name, - 'max-bitrate', - layer.maxBitrate, - ); - enc.maxBitrate = layer.maxBitrate; - changed = true; - } - - if ( - layer.maxFramerate > 0 && - layer.maxFramerate !== enc.maxFramerate - ) { - this.logger( - 'debug', - '[dynascale]: setting maxFramerate from server', - 'layer', - layer.name, - 'max-framerate', - layer.maxFramerate, - ); - enc.maxFramerate = layer.maxFramerate; - changed = true; - } - } + // skip the rest of the settings if the layer is disabled + if (!shouldActivate) continue; + + const { + maxFramerate, + scaleResolutionDownBy, + maxBitrate, + scalabilityMode, + } = layer; + if ( + scaleResolutionDownBy >= 1 && + scaleResolutionDownBy !== encoder.scaleResolutionDownBy + ) { + encoder.scaleResolutionDownBy = scaleResolutionDownBy; + changed = true; } - }); + if (maxBitrate > 0 && maxBitrate !== encoder.maxBitrate) { + encoder.maxBitrate = maxBitrate; + changed = true; + } + if (maxFramerate > 0 && maxFramerate !== encoder.maxFramerate) { + encoder.maxFramerate = maxFramerate; + changed = true; + } + // @ts-expect-error scalabilityMode is not in the typedefs yet + if (scalabilityMode && scalabilityMode !== encoder.scalabilityMode) { + // @ts-expect-error scalabilityMode is not in the typedefs yet + encoder.scalabilityMode = scalabilityMode; + changed = true; + } + } const activeLayers = params.encodings.filter((e) => e.active); - if (changed) { - await videoSender.setParameters(params); - this.logger( - 'info', - `Update publish quality, enabled rids: `, - activeLayers, - ); - } else { - this.logger('info', `Update publish quality, no change: `, activeLayers); + if (!changed) { + this.logger('info', `Update publish quality, no change:`, activeLayers); + return; } + + await videoSender.setParameters(params); + this.logger('info', `Update publish quality, enabled rids:`, activeLayers); }; /** diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index c275ab7809..9bcd72d9a5 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -64,7 +64,7 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { if (!call) throw new Error('No active call found'); try { const prontoDefaultCodec = - isProntoEnvironment && !isFirefox ? 'h264' : 'vp8'; + isProntoEnvironment && !isFirefox ? 'vp9' : 'vp8'; const preferredCodec = videoCodecOverride || prontoDefaultCodec; const videoSettings = call.state.settings?.video; diff --git a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts b/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts index 9ae414c060..4175e84b26 100644 --- a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts +++ b/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts @@ -1,4 +1,6 @@ -const lookup: Record | undefined> = { +import { PreferredCodec } from '@stream-io/video-react-sdk'; + +const lookup: Record | undefined> = { h264: { 1080: 2_750_000, 720: 1_250_000, @@ -11,6 +13,18 @@ const lookup: Record | undefined> = { 540: 600_000, 360: 350_000, }, + vp9: { + 1080: 1_200_000, + 720: 750_000, + 540: 450_000, + 360: 275_000, + }, + av1: { + 1080: 1_000_000, + 720: 600_000, + 540: 350_000, + 360: 200_000, + }, }; export const getPreferredBitrate = ( From 2d10990125378227db6aaa05763c1e085c053074 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 8 Oct 2024 18:35:18 +0200 Subject: [PATCH 10/33] feat: for SVC codecs send one layer but announce three --- packages/client/src/rtc/Publisher.ts | 9 ++++++--- packages/client/src/rtc/__tests__/Publisher.test.ts | 1 + packages/client/src/rtc/__tests__/videoLayers.test.ts | 6 ++++-- packages/client/src/rtc/codecs.ts | 9 +++++++++ packages/client/src/rtc/videoLayers.ts | 8 ++++---- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index d7abf2682f..847f26173b 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -13,7 +13,7 @@ import { findOptimalVideoLayers, OptimalVideoLayer, } from './videoLayers'; -import { getPreferredCodecs, getRNOptimalCodec } from './codecs'; +import { getPreferredCodecs, getRNOptimalCodec, isSvcCodec } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; import { CallingState, CallState } from '../store'; import { PublishOptions } from '../types'; @@ -292,13 +292,17 @@ export class Publisher { track.enabled = true; } + const { preferredCodec } = opts; + const svcCodec = isSvcCodec(preferredCodec); transceiver = this.pc.addTransceiver(track, { direction: 'sendonly', streams: trackType === TrackType.VIDEO || trackType === TrackType.SCREEN_SHARE ? [mediaStream] : undefined, - sendEncodings: videoEncodings, + sendEncodings: svcCodec + ? videoEncodings?.filter((l) => l.rid === 'q') + : videoEncodings, }); this.logger('debug', `Added ${TrackType[trackType]} transceiver`); @@ -306,7 +310,6 @@ export class Publisher { this.transceiverRegistry[trackType] = transceiver; this.publishOptionsPerTrackType.set(trackType, opts); - const { preferredCodec } = opts; const codec = isReactNative() && trackType === TrackType.VIDEO && !preferredCodec ? getRNOptimalCodec() diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 1777a5e3d3..f81c899c4f 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -27,6 +27,7 @@ vi.mock('../codecs', () => { sdpFmtpLine: 'profile-level-id=42e01f', }, ]), + isSvcCodec: vi.fn(() => false), }; }); diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index 59764b2c58..593c9c5cb3 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -145,9 +145,11 @@ describe('videoLayers', () => { preferredCodec: 'vp9', scalabilityMode: 'L3T3', }); - expect(layers.length).toBe(1); - expect(layers[0].rid).toBe('q'); + expect(layers.length).toBe(3); expect(layers[0].scalabilityMode).toBe('L3T3'); + expect(layers[0].rid).toBe('q'); + expect(layers[1].rid).toBe('h'); + expect(layers[2].rid).toBe('f'); }); describe('getComputedMaxBitrate', () => { diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 3e46db9751..327a615460 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -103,3 +103,12 @@ export const getRNOptimalCodec = () => { if (osName === 'android') return 'vp8'; return undefined; }; + +/** + * Returns whether the codec is an SVC codec. + * + * @param codec the codec to check. + */ +export const isSvcCodec = (codec: string | undefined | null) => { + return codec === 'vp9' || codec === 'av1'; +}; diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 1a3394f5b3..c1d2920843 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,5 +1,6 @@ import { PublishOptions } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; +import { isSvcCodec } from './codecs'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { width: number; @@ -51,9 +52,8 @@ export const findOptimalVideoLayers = ( ); let downscaleFactor = 1; let bitrateFactor = 1; - const isSvcCodec = preferredCodec === 'vp9' || preferredCodec === 'av1'; - const layers = isSvcCodec ? ['q'] : ['f', 'h', 'q']; - for (const rid of layers) { + const svcCodec = isSvcCodec(preferredCodec); + for (const rid of ['f', 'h', 'q']) { const layer: OptimalVideoLayer = { active: true, rid, @@ -64,7 +64,7 @@ export const findOptimalVideoLayers = ( scaleResolutionDownBy: downscaleFactor, maxFramerate: 30, }; - if (isSvcCodec) { + if (svcCodec) { layer.scalabilityMode = scalabilityMode || 'L3T3_KEY'; } From f2d46c1a4c6eea549076e24fd5d57f64ff0c888f Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 8 Oct 2024 18:41:02 +0200 Subject: [PATCH 11/33] feat: remap `f` to `q` --- packages/client/src/rtc/Publisher.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 847f26173b..eb85a23649 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -301,7 +301,9 @@ export class Publisher { ? [mediaStream] : undefined, sendEncodings: svcCodec - ? videoEncodings?.filter((l) => l.rid === 'q') + ? videoEncodings + ?.filter((l) => l.rid === 'f') + .map((l) => ({ ...l, rid: 'q' })) : videoEncodings, }); From fef5dadba1251213215bf651ddf831553a691cfe Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 8 Oct 2024 18:57:34 +0200 Subject: [PATCH 12/33] feat: improve bitrates --- sample-apps/react/react-dogfood/helpers/bitrateLookup.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts b/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts index 4175e84b26..5530a5f59a 100644 --- a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts +++ b/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts @@ -14,8 +14,8 @@ const lookup: Record | undefined> = { 360: 350_000, }, vp9: { - 1080: 1_200_000, - 720: 750_000, + 1080: 1_250_000, + 720: 900_000, 540: 450_000, 360: 275_000, }, From bca74ea7e8cfbd301b64c8f9064f2da5f3be9285 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 8 Oct 2024 19:02:15 +0200 Subject: [PATCH 13/33] feat: improve bitrates --- sample-apps/react/react-dogfood/helpers/bitrateLookup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts b/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts index 5530a5f59a..880dbe00a5 100644 --- a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts +++ b/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts @@ -15,7 +15,7 @@ const lookup: Record | undefined> = { }, vp9: { 1080: 1_250_000, - 720: 900_000, + 720: 950_000, 540: 450_000, 360: 275_000, }, From ece6370234b4fb116a9299ce939ddb4fa61afaf2 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 10 Oct 2024 22:55:04 +0200 Subject: [PATCH 14/33] fix: separate svc from simulcast handling --- packages/client/src/rtc/Publisher.ts | 22 ++++++++++++++-------- packages/client/src/rtc/codecs.ts | 13 ++++++++++--- packages/client/src/rtc/videoLayers.ts | 21 ++++++++++++++------- packages/client/src/types.ts | 19 ++++++++++++++++++- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index eb85a23649..0e46d0147c 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -293,17 +293,17 @@ export class Publisher { } const { preferredCodec } = opts; - const svcCodec = isSvcCodec(preferredCodec); + const usesSvcCodec = isSvcCodec(preferredCodec); transceiver = this.pc.addTransceiver(track, { direction: 'sendonly', streams: trackType === TrackType.VIDEO || trackType === TrackType.SCREEN_SHARE ? [mediaStream] : undefined, - sendEncodings: svcCodec + sendEncodings: usesSvcCodec ? videoEncodings ?.filter((l) => l.rid === 'f') - .map((l) => ({ ...l, rid: 'q' })) + .map((l) => ({ ...l, rid: 'q' })) // downgrade the highest layer to 'q' : videoEncodings, }); @@ -451,20 +451,26 @@ export class Publisher { return; } + const [codecInUse] = params.codecs; + const usesSvcCodec = isSvcCodec(codecInUse.mimeType); + let changed = false; for (const encoder of params.encodings) { - const layer = enabledLayers.find((vls) => vls.name === encoder.rid); - if (!layer) continue; + const layer = usesSvcCodec + ? // for SVC, we only have one layer (q) and often rid is omitted + enabledLayers[0] + : // for non-SVC, we need to find the layer by rid (simulcast) + enabledLayers.find((l) => l.name === encoder.rid); // flip 'active' flag only when necessary - const shouldActivate = layer.active; + const shouldActivate = layer?.active; if (shouldActivate !== encoder.active) { encoder.active = shouldActivate; changed = true; } - // skip the rest of the settings if the layer is disabled - if (!shouldActivate) continue; + // skip the rest of the settings if the layer is disabled or not found + if (!layer) continue; const { maxFramerate, diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 327a615460..0afd79fe73 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -107,8 +107,15 @@ export const getRNOptimalCodec = () => { /** * Returns whether the codec is an SVC codec. * - * @param codec the codec to check. + * @param codecOrMimeType the codec to check. */ -export const isSvcCodec = (codec: string | undefined | null) => { - return codec === 'vp9' || codec === 'av1'; +export const isSvcCodec = (codecOrMimeType: string | undefined) => { + if (!codecOrMimeType) return false; + codecOrMimeType = codecOrMimeType.toLowerCase(); + return ( + codecOrMimeType === 'vp9' || + codecOrMimeType === 'av1' || + codecOrMimeType === 'video/vp9' || + codecOrMimeType === 'video/av1' + ); }; diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index c1d2920843..d748e93e93 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -57,23 +57,30 @@ export const findOptimalVideoLayers = ( const layer: OptimalVideoLayer = { active: true, rid, - width: Math.round(width / downscaleFactor), - height: Math.round(height / downscaleFactor), - maxBitrate: - Math.round(maxBitrate / bitrateFactor) || defaultBitratePerRid[rid], - scaleResolutionDownBy: downscaleFactor, + width, + height, + maxBitrate, maxFramerate: 30, }; if (svcCodec) { + // for SVC codecs, we need to set the scalability mode, and the + // codec will handle the rest (layers, temporal layers, etc.) layer.scalabilityMode = scalabilityMode || 'L3T3_KEY'; + } else { + // for non-SVC codecs, we need to downscale proportionally (simulcast) + layer.width = Math.round(width / downscaleFactor); + layer.height = Math.round(height / downscaleFactor); + const bitrate = Math.round(maxBitrate / bitrateFactor); + layer.maxBitrate = bitrate || defaultBitratePerRid[rid]; + layer.scaleResolutionDownBy = downscaleFactor; + downscaleFactor *= 2; + bitrateFactor *= bitrateDownscaleFactor; } // Reversing the order [f, h, q] to [q, h, f] as Chrome uses encoding index // when deciding which layer to disable when CPU or bandwidth is constrained. // Encodings should be ordered in increasing spatial resolution order. optimalVideoLayers.unshift(layer); - downscaleFactor *= 2; - bitrateFactor *= bitrateDownscaleFactor; } // for simplicity, we start with all layers enabled, then this function diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index b495b1f434..546e9872a4 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -153,10 +153,27 @@ export type SubscriptionChanges = { export type PreferredCodec = 'vp8' | 'h264' | 'vp9' | 'av1' | string; export type PublishOptions = { - preferredCodec?: PreferredCodec | null; + /** + * The preferred codec to use when publishing the video stream. + */ + preferredCodec?: PreferredCodec; + /** + * The preferred scalability to use when publishing the video stream. + * Applicable only for SVC codecs. + */ scalabilityMode?: string; + /** + * The preferred bitrate to use when publishing the video stream. + */ preferredBitrate?: number; + /** + * The preferred downscale factor to use when publishing the video stream + * in simulcast mode (non-SVC). + */ bitrateDownscaleFactor?: number; + /** + * Screen share settings. + */ screenShareSettings?: ScreenShareSettings; }; From ce5ffaa658d50cbb32679b132ad83acdd7fdcb60 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 10 Oct 2024 23:33:06 +0200 Subject: [PATCH 15/33] tests for changePublishQuality --- packages/client/src/rtc/Publisher.ts | 2 +- .../src/rtc/__tests__/Publisher.test.ts | 141 ++++++++++++++++++ .../src/rtc/__tests__/mocks/webrtc.mocks.ts | 2 + 3 files changed, 144 insertions(+), 1 deletion(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 0e46d0147c..3f9f151969 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -452,7 +452,7 @@ export class Publisher { } const [codecInUse] = params.codecs; - const usesSvcCodec = isSvcCodec(codecInUse.mimeType); + const usesSvcCodec = codecInUse && isSvcCodec(codecInUse.mimeType); let changed = false; for (const encoder of params.encodings) { diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index f81c899c4f..0e0ea10894 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -278,4 +278,145 @@ describe('Publisher', () => { expect(publisher.restartIce).toHaveBeenCalled(); }); }); + + describe('changePublishQuality', () => { + it('can dynamically activate/deactivate simulcast layers', async () => { + const transceiver = new RTCRtpTransceiver(); + const setParametersSpy = vi + .spyOn(transceiver.sender, 'setParameters') + .mockResolvedValue(); + const getParametersSpy = vi + .spyOn(transceiver.sender, 'getParameters') + .mockReturnValue({ + codecs: [ + // @ts-expect-error incomplete data + { mimeType: 'video/VP8' }, + // @ts-expect-error incomplete data + { mimeType: 'video/VP9' }, + // @ts-expect-error incomplete data + { mimeType: 'video/H264' }, + // @ts-expect-error incomplete data + { mimeType: 'video/AV1' }, + ], + encodings: [ + { rid: 'q', active: true }, + { rid: 'h', active: true }, + { rid: 'f', active: true }, + ], + }); + + // inject the transceiver + publisher['transceiverRegistry'][TrackType.VIDEO] = transceiver; + + await publisher['changePublishQuality']([ + { + name: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 4, + maxFramerate: 30, + scalabilityMode: '', + }, + { + name: 'h', + active: false, + maxBitrate: 150, + scaleResolutionDownBy: 2, + maxFramerate: 30, + scalabilityMode: '', + }, + { + name: 'f', + active: true, + maxBitrate: 200, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: '', + }, + ]); + + expect(getParametersSpy).toHaveBeenCalled(); + expect(setParametersSpy).toHaveBeenCalled(); + expect(setParametersSpy.mock.calls[0][0].encodings).toEqual([ + { + rid: 'q', + active: true, + maxBitrate: 100, + scaleResolutionDownBy: 4, + maxFramerate: 30, + }, + { + rid: 'h', + active: false, + maxBitrate: 150, + scaleResolutionDownBy: 2, + maxFramerate: 30, + }, + { + rid: 'f', + active: true, + maxBitrate: 200, + scaleResolutionDownBy: 1, + maxFramerate: 30, + }, + ]); + }); + + it('can dynamically update scalability mode in SVC', async () => { + const transceiver = new RTCRtpTransceiver(); + const setParametersSpy = vi + .spyOn(transceiver.sender, 'setParameters') + .mockResolvedValue(); + const getParametersSpy = vi + .spyOn(transceiver.sender, 'getParameters') + .mockReturnValue({ + codecs: [ + // @ts-expect-error incomplete data + { mimeType: 'video/VP9' }, + // @ts-expect-error incomplete data + { mimeType: 'video/AV1' }, + // @ts-expect-error incomplete data + { mimeType: 'video/VP8' }, + // @ts-expect-error incomplete data + { mimeType: 'video/H264' }, + ], + encodings: [ + { + rid: 'q', + active: true, + maxBitrate: 100, + // @ts-expect-error not in the standard lib yet + scalabilityMode: 'L3T3_KEY', + }, + ], + }); + + // inject the transceiver + publisher['transceiverRegistry'][TrackType.VIDEO] = transceiver; + + await publisher['changePublishQuality']([ + { + name: 'q', + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ]); + + expect(getParametersSpy).toHaveBeenCalled(); + expect(setParametersSpy).toHaveBeenCalled(); + expect(setParametersSpy.mock.calls[0][0].encodings).toEqual([ + { + rid: 'q', + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ]); + }); + }); }); diff --git a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts index 8143bfc4a7..5a4424fbba 100644 --- a/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts +++ b/packages/client/src/rtc/__tests__/mocks/webrtc.mocks.ts @@ -45,6 +45,8 @@ const RTCRtpTransceiverMock = vi.fn((): Partial => { sender: { track: null, replaceTrack: vi.fn(), + getParameters: vi.fn(), + setParameters: vi.fn(), }, setCodecPreferences: vi.fn(), }; From da0e4fb33d0c7441810f62e9efa55bfc0f5e9c23 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 10 Oct 2024 23:58:06 +0200 Subject: [PATCH 16/33] tests for changePublishQuality --- .../src/rtc/__tests__/Publisher.test.ts | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index 0e0ea10894..f9e8e885e1 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -17,7 +17,8 @@ vi.mock('../../StreamSfuClient', () => { }; }); -vi.mock('../codecs', () => { +vi.mock('../codecs', async () => { + const codecs = await vi.importActual('../codecs'); return { getPreferredCodecs: vi.fn((): RTCRtpCodecCapability[] => [ { @@ -27,7 +28,7 @@ vi.mock('../codecs', () => { sdpFmtpLine: 'profile-level-id=42e01f', }, ]), - isSvcCodec: vi.fn(() => false), + isSvcCodec: codecs.isSvcCodec, }; }); @@ -418,5 +419,54 @@ describe('Publisher', () => { }, ]); }); + + it('supports empty rid in SVC', async () => { + const transceiver = new RTCRtpTransceiver(); + const setParametersSpy = vi + .spyOn(transceiver.sender, 'setParameters') + .mockResolvedValue(); + const getParametersSpy = vi + .spyOn(transceiver.sender, 'getParameters') + .mockReturnValue({ + codecs: [ + // @ts-expect-error incomplete data + { mimeType: 'video/VP9' }, + ], + encodings: [ + { + rid: undefined, // empty rid + active: true, + // @ts-expect-error not in the standard lib yet + scalabilityMode: 'L3T3_KEY', + }, + ], + }); + + // inject the transceiver + publisher['transceiverRegistry'][TrackType.VIDEO] = transceiver; + + await publisher['changePublishQuality']([ + { + name: 'q', + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ]); + + expect(getParametersSpy).toHaveBeenCalled(); + expect(setParametersSpy).toHaveBeenCalled(); + expect(setParametersSpy.mock.calls[0][0].encodings).toEqual([ + { + active: true, + maxBitrate: 50, + scaleResolutionDownBy: 1, + maxFramerate: 30, + scalabilityMode: 'L1T3', + }, + ]); + }); }); }); From 9770def6b0f03e4263f2fb790d86f13316e0788c Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Thu, 10 Oct 2024 23:59:01 +0200 Subject: [PATCH 17/33] chore: simplify type --- packages/client/src/rtc/Publisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 3f9f151969..7118755847 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -522,7 +522,7 @@ export class Publisher { private getCodecPreferences = ( trackType: TrackType, - preferredCodec?: string | null, + preferredCodec?: string, ) => { if (trackType === TrackType.VIDEO) { return getPreferredCodecs('video', preferredCodec || 'vp8'); From 89afbb10f68582cd29538c79b493646431116825 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Fri, 11 Oct 2024 14:19:40 +0200 Subject: [PATCH 18/33] fix: use boolean --- packages/client/src/rtc/Publisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 7118755847..8f7601dca7 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -463,7 +463,7 @@ export class Publisher { enabledLayers.find((l) => l.name === encoder.rid); // flip 'active' flag only when necessary - const shouldActivate = layer?.active; + const shouldActivate = !!layer?.active; if (shouldActivate !== encoder.active) { encoder.active = shouldActivate; changed = true; From 01a5ceca716a21dfa3fe8dd7a76276d53dc03a1c Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Mon, 14 Oct 2024 14:42:13 +0200 Subject: [PATCH 19/33] feat: move bitrate lookup to the SDK --- packages/client/src/devices/CameraManager.ts | 18 ----- packages/client/src/rtc/Publisher.ts | 79 ++++++------------- .../src/rtc/__tests__/Publisher.test.ts | 7 +- .../src/rtc/__tests__/bitrateLookup.test.ts | 12 +++ packages/client/src/rtc/bitrateLookup.ts | 61 ++++++++++++++ packages/client/src/rtc/codecs.ts | 23 +++--- packages/client/src/rtc/videoLayers.ts | 19 +++-- .../react-dogfood/components/MeetingUI.tsx | 10 +-- .../react-dogfood/helpers/bitrateLookup.ts | 38 --------- 9 files changed, 130 insertions(+), 137 deletions(-) create mode 100644 packages/client/src/rtc/__tests__/bitrateLookup.test.ts create mode 100644 packages/client/src/rtc/bitrateLookup.ts delete mode 100644 sample-apps/react/react-dogfood/helpers/bitrateLookup.ts diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index f82aef4eb7..259f3bf59e 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -102,24 +102,6 @@ export class CameraManager extends InputMediaDeviceManager { this.publishOptions = { ...this.publishOptions, ...options }; } - /** - * Returns the capture resolution of the camera. - */ - getCaptureResolution() { - const { mediaStream } = this.state; - if (!mediaStream) return; - - const [videoTrack] = mediaStream.getVideoTracks(); - if (!videoTrack) return; - - const settings = videoTrack.getSettings(); - return { - width: settings.width, - height: settings.height, - frameRate: settings.frameRate, - }; - } - protected getDevices(): Observable { return getVideoDevices(); } diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 8f7601dca7..cb180dcae7 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -13,11 +13,10 @@ import { findOptimalVideoLayers, OptimalVideoLayer, } from './videoLayers'; -import { getPreferredCodecs, getRNOptimalCodec, isSvcCodec } from './codecs'; +import { getOptimalVideoCodec, getPreferredCodecs, isSvcCodec } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; import { CallingState, CallState } from '../store'; import { PublishOptions } from '../types'; -import { isReactNative } from '../helpers/platforms'; import { enableHighQualityAudio, toggleDtx } from '../helpers/sdp-munging'; import { Logger } from '../coordinator/connection/types'; import { getLogger } from '../logger'; @@ -46,21 +45,9 @@ export class Publisher { private readonly logger: Logger; private pc: RTCPeerConnection; private readonly state: CallState; - - private readonly transceiverRegistry: { - [key in TrackType]: RTCRtpTransceiver | undefined; - } = { - [TrackType.AUDIO]: undefined, - [TrackType.VIDEO]: undefined, - [TrackType.SCREEN_SHARE]: undefined, - [TrackType.SCREEN_SHARE_AUDIO]: undefined, - [TrackType.UNSPECIFIED]: undefined, - }; - - private readonly publishOptionsPerTrackType = new Map< - TrackType, - PublishOptions - >(); + private readonly transceiverCache = new Map(); + private readonly trackLayersCache = new Map(); + private readonly publishOptsForTrack = new Map(); /** * An array maintaining the order how transceivers were added to the peer connection. @@ -81,16 +68,6 @@ export class Publisher { [TrackType.UNSPECIFIED]: undefined, }; - private readonly trackLayersCache: { - [key in TrackType]: OptimalVideoLayer[] | undefined; - } = { - [TrackType.AUDIO]: undefined, - [TrackType.VIDEO]: undefined, - [TrackType.SCREEN_SHARE]: undefined, - [TrackType.SCREEN_SHARE_AUDIO]: undefined, - [TrackType.UNSPECIFIED]: undefined, - }; - private readonly isDtxEnabled: boolean; private readonly isRedEnabled: boolean; @@ -184,14 +161,8 @@ export class Publisher { close = ({ stopTracks }: { stopTracks: boolean }) => { if (stopTracks) { this.stopPublishing(); - Object.keys(this.transceiverRegistry).forEach((trackType) => { - // @ts-ignore - this.transceiverRegistry[trackType] = undefined; - }); - Object.keys(this.trackLayersCache).forEach((trackType) => { - // @ts-ignore - this.trackLayersCache[trackType] = undefined; - }); + this.transceiverCache.clear(); + this.trackLayersCache.clear(); } this.detachEventHandlers(); @@ -249,7 +220,7 @@ export class Publisher { .getTransceivers() .find( (t) => - t === this.transceiverRegistry[trackType] && + t === this.transceiverCache.get(trackType) && t.sender.track && t.sender.track?.kind === this.trackKindMapping[trackType], ); @@ -303,18 +274,18 @@ export class Publisher { sendEncodings: usesSvcCodec ? videoEncodings ?.filter((l) => l.rid === 'f') - .map((l) => ({ ...l, rid: 'q' })) // downgrade the highest layer to 'q' + .map((l) => ({ ...l, rid: 'q' })) // announce the 'f' layer as 'q' : videoEncodings, }); this.logger('debug', `Added ${TrackType[trackType]} transceiver`); this.transceiverInitOrder.push(trackType); - this.transceiverRegistry[trackType] = transceiver; - this.publishOptionsPerTrackType.set(trackType, opts); + this.transceiverCache.set(trackType, transceiver); + this.publishOptsForTrack.set(trackType, opts); const codec = - isReactNative() && trackType === TrackType.VIDEO && !preferredCodec - ? getRNOptimalCodec() + trackType === TrackType.VIDEO && !preferredCodec + ? getOptimalVideoCodec() : preferredCodec; const codecPreferences = @@ -359,7 +330,9 @@ export class Publisher { unpublishStream = async (trackType: TrackType, stopTrack: boolean) => { const transceiver = this.pc .getTransceivers() - .find((t) => t === this.transceiverRegistry[trackType] && t.sender.track); + .find( + (t) => t === this.transceiverCache.get(trackType) && t.sender.track, + ); if ( transceiver && transceiver.sender.track && @@ -383,7 +356,7 @@ export class Publisher { * @param trackType the track type to check. */ isPublishing = (trackType: TrackType): boolean => { - const transceiver = this.transceiverRegistry[trackType]; + const transceiver = this.transceiverCache.get(trackType); if (!transceiver || !transceiver.sender) return false; const track = transceiver.sender.track; return !!track && track.readyState === 'live' && track.enabled; @@ -436,7 +409,7 @@ export class Publisher { enabledLayers, ); - const videoSender = this.transceiverRegistry[TrackType.VIDEO]?.sender; + const videoSender = this.transceiverCache.get(TrackType.VIDEO)?.sender; if (!videoSender) { this.logger('warn', 'Update publish quality, no video sender found.'); return; @@ -631,7 +604,7 @@ export class Publisher { }; private enableHighQualityAudio = (sdp: string) => { - const transceiver = this.transceiverRegistry[TrackType.SCREEN_SHARE_AUDIO]; + const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO); if (!transceiver) return sdp; const mid = this.extractMid(transceiver, sdp, TrackType.SCREEN_SHARE_AUDIO); @@ -704,27 +677,25 @@ export class Publisher { .getTransceivers() .filter((t) => t.direction === 'sendonly' && t.sender.track) .map((transceiver) => { - const trackType: TrackType = Number( - Object.keys(this.transceiverRegistry).find( - (key) => - this.transceiverRegistry[key as any as TrackType] === transceiver, - ), - ); + let trackType!: TrackType; + this.transceiverCache.forEach((value, key) => { + if (value === transceiver) trackType = key; + }); const track = transceiver.sender.track!; let optimalLayers: OptimalVideoLayer[]; const isTrackLive = track.readyState === 'live'; if (isTrackLive) { - const publishOpts = this.publishOptionsPerTrackType.get(trackType); + const publishOpts = this.publishOptsForTrack.get(trackType); optimalLayers = trackType === TrackType.VIDEO ? findOptimalVideoLayers(track, targetResolution, publishOpts) : trackType === TrackType.SCREEN_SHARE ? findOptimalScreenSharingLayers(track, publishOpts) : []; - this.trackLayersCache[trackType] = optimalLayers; + this.trackLayersCache.set(trackType, optimalLayers); } else { // we report the last known optimal layers for ended tracks - optimalLayers = this.trackLayersCache[trackType] || []; + optimalLayers = this.trackLayersCache.get(trackType) || []; this.logger( 'debug', `Track ${TrackType[trackType]} is ended. Announcing last known optimal layers`, diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index f9e8e885e1..c8d573a014 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -28,6 +28,7 @@ vi.mock('../codecs', async () => { sdpFmtpLine: 'profile-level-id=42e01f', }, ]), + getOptimalVideoCodec: codecs.getOptimalVideoCodec, isSvcCodec: codecs.isSvcCodec, }; }); @@ -307,7 +308,7 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverRegistry'][TrackType.VIDEO] = transceiver; + publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); await publisher['changePublishQuality']([ { @@ -393,7 +394,7 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverRegistry'][TrackType.VIDEO] = transceiver; + publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); await publisher['changePublishQuality']([ { @@ -443,7 +444,7 @@ describe('Publisher', () => { }); // inject the transceiver - publisher['transceiverRegistry'][TrackType.VIDEO] = transceiver; + publisher['transceiverCache'].set(TrackType.VIDEO, transceiver); await publisher['changePublishQuality']([ { diff --git a/packages/client/src/rtc/__tests__/bitrateLookup.test.ts b/packages/client/src/rtc/__tests__/bitrateLookup.test.ts new file mode 100644 index 0000000000..6ec4c67426 --- /dev/null +++ b/packages/client/src/rtc/__tests__/bitrateLookup.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from 'vitest'; +import { getOptimalBitrate } from '../bitrateLookup'; + +describe('bitrateLookup', () => { + it('should return optimal bitrate', () => { + expect(getOptimalBitrate('vp9', 720)).toBe(1_250_000); + }); + + it('should return nearest bitrate for exotic dimensions', () => { + expect(getOptimalBitrate('vp9', 1000)).toBe(1_500_000); + }); +}); diff --git a/packages/client/src/rtc/bitrateLookup.ts b/packages/client/src/rtc/bitrateLookup.ts new file mode 100644 index 0000000000..8d961816f5 --- /dev/null +++ b/packages/client/src/rtc/bitrateLookup.ts @@ -0,0 +1,61 @@ +import { PreferredCodec } from '../types'; + +const bitrateLookupTable: Record< + PreferredCodec, + Record | undefined +> = { + h264: { + 2160: 5_000_000, + 1440: 3_500_000, + 1080: 2_750_000, + 720: 1_250_000, + 540: 750_000, + 360: 400_000, + default: 1_250_000, + }, + vp8: { + 2160: 5_000_000, + 1440: 2_750_000, + 1080: 2_000_000, + 720: 1_250_000, + 540: 600_000, + 360: 350_000, + default: 1_250_000, + }, + vp9: { + 2160: 3_000_000, + 1440: 2_000_000, + 1080: 1_500_000, + 720: 1_250_000, + 540: 500_000, + 360: 275_000, + default: 1_250_000, + }, + av1: { + 2160: 2_000_000, + 1440: 1_550_000, + 1080: 1_000_000, + 720: 600_000, + 540: 350_000, + 360: 200_000, + default: 600_000, + }, +}; + +export const getOptimalBitrate = ( + codec: string, + frameHeight: number, +): number => { + const codecLookup = bitrateLookupTable[codec.toLowerCase()]; + if (!codecLookup) throw new Error(`Unknown codec: ${codec}`); + + let bitrate = codecLookup[frameHeight]; + if (!bitrate) { + const keys = Object.keys(codecLookup).map(Number); + const nearest = keys.reduce((a, b) => + Math.abs(b - frameHeight) < Math.abs(a - frameHeight) ? b : a, + ); + bitrate = codecLookup[nearest]; + } + return bitrate ?? codecLookup.default!; +}; diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 0afd79fe73..c7ad0969ef 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -1,4 +1,7 @@ import { getOSInfo } from '../client-details'; +import { isReactNative } from '../helpers/platforms'; +import { isFirefox, isSafari } from '../helpers/browsers'; +import type { PreferredCodec } from '../types'; /** * Returns back a list of sorted codecs, with the preferred codec first. @@ -92,16 +95,18 @@ export const getGenericSdp = async (direction: RTCRtpTransceiverDirection) => { }; /** - * Returns the optimal codec for RN. + * Returns the optimal video codec for the device. */ -export const getRNOptimalCodec = () => { - const osName = getOSInfo()?.name.toLowerCase(); - // in ipads it was noticed that if vp8 codec is used - // then the bytes sent is 0 in the outbound-rtp - // so we are forcing h264 codec for ipads - if (osName === 'ipados') return 'h264'; - if (osName === 'android') return 'vp8'; - return undefined; +export const getOptimalVideoCodec = (): PreferredCodec => { + if (isReactNative()) { + const osName = getOSInfo()?.name.toLowerCase(); + if (osName === 'ios' || osName === 'ipados') return 'h264'; + if (osName === 'android') return 'vp8'; // TODO switch to vp9 + return 'vp8'; + } + if (isSafari()) return 'h264'; + if (isFirefox()) return 'vp8'; + return 'vp8'; // TODO switch to vp9 }; /** diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index d748e93e93..9ac9bd90d3 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,6 +1,7 @@ import { PublishOptions } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; import { isSvcCodec } from './codecs'; +import { getOptimalBitrate } from './bitrateLookup'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { width: number; @@ -41,14 +42,13 @@ export const findOptimalVideoLayers = ( const { preferredCodec, scalabilityMode, - preferredBitrate, bitrateDownscaleFactor = 2, } = publishOptions || {}; const maxBitrate = getComputedMaxBitrate( targetResolution, width, height, - preferredBitrate, + publishOptions, ); let downscaleFactor = 1; let bitrateFactor = 1; @@ -65,7 +65,7 @@ export const findOptimalVideoLayers = ( if (svcCodec) { // for SVC codecs, we need to set the scalability mode, and the // codec will handle the rest (layers, temporal layers, etc.) - layer.scalabilityMode = scalabilityMode || 'L3T3_KEY'; + layer.scalabilityMode = scalabilityMode || 'L3T2_KEY'; } else { // for non-SVC codecs, we need to downscale proportionally (simulcast) layer.width = Math.round(width / downscaleFactor); @@ -98,13 +98,13 @@ export const findOptimalVideoLayers = ( * @param targetResolution the target resolution. * @param currentWidth the current width of the track. * @param currentHeight the current height of the track. - * @param preferredBitrate the preferred bitrate for the track. + * @param publishOptions the publish options. */ export const getComputedMaxBitrate = ( targetResolution: TargetResolutionResponse, currentWidth: number, currentHeight: number, - preferredBitrate: number | undefined, + publishOptions: PublishOptions | undefined, ): number => { // if the current resolution is lower than the target resolution, // we want to proportionally reduce the target bitrate @@ -113,7 +113,14 @@ export const getComputedMaxBitrate = ( height: targetHeight, bitrate: targetBitrate, } = targetResolution; - const bitrate = preferredBitrate || targetBitrate; + const { preferredBitrate, preferredCodec } = publishOptions || {}; + const frameHeight = + currentWidth > currentHeight ? currentHeight : currentWidth; + const bitrate = + preferredBitrate || + (preferredCodec + ? getOptimalBitrate(preferredCodec, frameHeight) + : targetBitrate); if (currentWidth < targetWidth || currentHeight < targetHeight) { const currentPixels = currentWidth * currentHeight; const targetPixels = targetWidth * targetHeight; diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 9bcd72d9a5..b46c081f17 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -22,7 +22,6 @@ import { import { ActiveCall } from './ActiveCall'; import { Feedback } from './Feedback/Feedback'; import { DefaultAppHeader } from './DefaultAppHeader'; -import { getPreferredBitrate } from '../helpers/bitrateLookup'; import { useIsProntoEnvironment } from '../context/AppEnvironmentContext'; const contents = { @@ -66,16 +65,9 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const prontoDefaultCodec = isProntoEnvironment && !isFirefox ? 'vp9' : 'vp8'; const preferredCodec = videoCodecOverride || prontoDefaultCodec; - - const videoSettings = call.state.settings?.video; - const frameHeight = - call.camera.getCaptureResolution()?.height ?? - videoSettings?.target_resolution.height ?? - 1080; - const preferredBitrate = bitrateOverride ? parseInt(bitrateOverride, 10) - : getPreferredBitrate(preferredCodec, frameHeight); + : undefined; call.camera.updatePublishOptions({ preferredCodec, diff --git a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts b/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts deleted file mode 100644 index 880dbe00a5..0000000000 --- a/sample-apps/react/react-dogfood/helpers/bitrateLookup.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { PreferredCodec } from '@stream-io/video-react-sdk'; - -const lookup: Record | undefined> = { - h264: { - 1080: 2_750_000, - 720: 1_250_000, - 540: 750_000, - 360: 400_000, - }, - vp8: { - 1080: 2_000_000, - 720: 1_000_000, - 540: 600_000, - 360: 350_000, - }, - vp9: { - 1080: 1_250_000, - 720: 950_000, - 540: 450_000, - 360: 275_000, - }, - av1: { - 1080: 1_000_000, - 720: 600_000, - 540: 350_000, - 360: 200_000, - }, -}; - -export const getPreferredBitrate = ( - codec: string, - frameHeight: number, -): number | undefined => { - const codecLookup = lookup[codec.toLowerCase()]; - if (!codecLookup) return; - - return codecLookup[frameHeight]; -}; From d1f4b5e07bd53ea3a9418a60370ba900a8eb3815 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 15 Oct 2024 13:36:31 +0200 Subject: [PATCH 20/33] feat: allow guest logins in Pronto --- .../react/react-dogfood/pages/api/auth/[...nextauth].ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sample-apps/react/react-dogfood/pages/api/auth/[...nextauth].ts b/sample-apps/react/react-dogfood/pages/api/auth/[...nextauth].ts index 9dbb019d85..7ee5a3535a 100644 --- a/sample-apps/react/react-dogfood/pages/api/auth/[...nextauth].ts +++ b/sample-apps/react/react-dogfood/pages/api/auth/[...nextauth].ts @@ -25,7 +25,7 @@ const StreamDemoAccountProvider: CredentialsConfig }, authorize: async (credentials) => { const name = credentials?.name || names.random(); - const id = name.replace(/[^_\-0-9a-zA-Z@]/g, '_'); + const id = name.replace(/[^_\-0-9a-zA-Z@]/g, '_').replace(/ /g, '_'); return { id, name, @@ -41,7 +41,7 @@ const basePath = process.env.NEXT_PUBLIC_BASE_PATH || ''; export const authOptions: NextAuthOptions = { providers: [ - !isProntoEnvironment && StreamDemoAccountProvider, + StreamDemoAccountProvider, isProntoEnvironment && GoogleProvider({ clientId: process.env.GOOGLE_CLIENT_ID!, @@ -57,10 +57,7 @@ export const authOptions: NextAuthOptions = { const isStreamEmployee = email.endsWith('@getstream.io'); return email_verified && isStreamEmployee; } - return ( - !isProntoEnvironment && - account?.provider === StreamDemoAccountProvider.id - ); + return account?.provider === StreamDemoAccountProvider.id; }, async redirect({ baseUrl, url }) { // when running the demo on Vercel, we need to patch the baseUrl From a9427af29724d11d1ca3794c86bd42797f0c5cfb Mon Sep 17 00:00:00 2001 From: Santhosh Vaiyapuri Date: Tue, 15 Oct 2024 16:49:54 +0200 Subject: [PATCH 21/33] chore: update to rn-webrtc 124 --- .../react-native/dogfood/ios/Podfile.lock | 28 +++++++++---------- sample-apps/react-native/dogfood/package.json | 2 +- yarn.lock | 15 +++++++++- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/sample-apps/react-native/dogfood/ios/Podfile.lock b/sample-apps/react-native/dogfood/ios/Podfile.lock index 6280c8de9d..43982aae24 100644 --- a/sample-apps/react-native/dogfood/ios/Podfile.lock +++ b/sample-apps/react-native/dogfood/ios/Podfile.lock @@ -24,15 +24,15 @@ PODS: - GTMAppAuth (4.1.1): - AppAuth/Core (~> 1.7) - GTMSessionFetcher/Core (< 4.0, >= 3.3) - - GTMSessionFetcher/Core (3.4.1) + - GTMSessionFetcher/Core (3.5.0) - hermes-engine (0.73.4): - hermes-engine/Pre-built (= 0.73.4) - hermes-engine/Pre-built (0.73.4) - - JitsiWebRTC (118.0.0) + - JitsiWebRTC (124.0.1) - libevent (2.1.12) - - MMKV (1.3.5): - - MMKVCore (~> 1.3.5) - - MMKVCore (1.3.5) + - MMKV (1.3.9): + - MMKVCore (~> 1.3.9) + - MMKVCore (1.3.9) - PromisesObjC (2.4.0) - PromisesSwift (2.4.0): - PromisesObjC (= 2.4.0) @@ -1145,10 +1145,10 @@ PODS: - RCT-Folly (= 2022.05.16.00) - React-Core - stream-react-native-webrtc - - stream-react-native-webrtc (118.1.0): - - JitsiWebRTC (~> 118.0.0) + - stream-react-native-webrtc (124.0.0-rc.1): + - JitsiWebRTC (~> 124.0.0) - React-Core - - stream-video-react-native (1.1.0): + - stream-video-react-native (1.1.4): - glog - RCT-Folly (= 2022.05.16.00) - React-Core @@ -1406,12 +1406,12 @@ SPEC CHECKSUMS: glog: c5d68082e772fa1c511173d6b30a9de2c05a69a2 GoogleSignIn: d4281ab6cf21542b1cfaff85c191f230b399d2db GTMAppAuth: f69bd07d68cd3b766125f7e072c45d7340dea0de - GTMSessionFetcher: 8000756fc1c19d2e5697b90311f7832d2e33f6cd + GTMSessionFetcher: 5aea5ba6bd522a239e236100971f10cb71b96ab6 hermes-engine: b2669ce35fc4ac14f523b307aff8896799829fe2 - JitsiWebRTC: 3a41671ef65a51d7204323814b055a2690b921c7 + JitsiWebRTC: d0ae5fd6a81e771bfd82c2ee6c6bb542ebd65ee8 libevent: 4049cae6c81cdb3654a443be001fb9bdceff7913 - MMKV: 506311d0494023c2f7e0b62cc1f31b7370fa3cfb - MMKVCore: 9e2e5fd529b64a9fe15f1a7afb3d73b2e27b4db9 + MMKV: 817ba1eea17421547e01e087285606eb270a8dcb + MMKVCore: af055b00e27d88cd92fad301c5fecd1ff9b26dd9 PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47 PromisesSwift: 9d77319bbe72ebf6d872900551f7eeba9bce2851 RCT-Folly: 7169b2b1c44399c76a47b5deaaba715eeeb476c0 @@ -1478,8 +1478,8 @@ SPEC CHECKSUMS: RNVoipPushNotification: 543e18f83089134a35e7f1d2eba4c8b1f7776b08 SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17 stream-io-video-filters-react-native: 8c345e6adf5164646696d45f9962c4199b2fe3b9 - stream-react-native-webrtc: 4ccf61161f77c57b9aa45f78cb7f69b7d91f3e9f - stream-video-react-native: ffc867d9eb2a97f0eaa8b4376322ed6e82a5383d + stream-react-native-webrtc: 1380525629fae21f4894535189573fb4ddb7984a + stream-video-react-native: 25880e3ff2889deca42fba87f6fa0c434fc6e5fa TOCropViewController: 80b8985ad794298fb69d3341de183f33d1853654 Yoga: 1b901a6d6eeba4e8a2e8f308f708691cdb5db312 diff --git a/sample-apps/react-native/dogfood/package.json b/sample-apps/react-native/dogfood/package.json index 2b4c0ed99a..aacc9a6206 100644 --- a/sample-apps/react-native/dogfood/package.json +++ b/sample-apps/react-native/dogfood/package.json @@ -30,7 +30,7 @@ "@react-navigation/native": "^6.1.10", "@react-navigation/native-stack": "^6.9.18", "@stream-io/flat-list-mvcp": "^0.10.3", - "@stream-io/react-native-webrtc": "118.1.0", + "@stream-io/react-native-webrtc": "124.0.0-rc.1", "@stream-io/video-filters-react-native": "workspace:^", "@stream-io/video-react-native-sdk": "workspace:^", "react": "18.2.0", diff --git a/yarn.lock b/yarn.lock index bbf280b456..4f28f1ddd7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7239,6 +7239,19 @@ __metadata: languageName: node linkType: hard +"@stream-io/react-native-webrtc@npm:124.0.0-rc.1": + version: 124.0.0-rc.1 + resolution: "@stream-io/react-native-webrtc@npm:124.0.0-rc.1" + dependencies: + base64-js: 1.5.1 + debug: 4.3.4 + event-target-shim: 6.0.2 + peerDependencies: + react-native: ">=0.60.0" + checksum: d39e7652518d9f3dd904b947667ff1e9d0d35d0603e8e4b262f2d20e326328335a19e8e252881e17af9a1c0c4446f356c4aa6dc4744be2ecc5911e0ae2be3f25 + languageName: node + linkType: hard + "@stream-io/stream-video-react-tutorial@workspace:sample-apps/react/stream-video-react-tutorial": version: 0.0.0-use.local resolution: "@stream-io/stream-video-react-tutorial@workspace:sample-apps/react/stream-video-react-tutorial" @@ -7455,7 +7468,7 @@ __metadata: "@rnx-kit/metro-config": ^1.3.3 "@rnx-kit/metro-resolver-symlinks": ^0.1.22 "@stream-io/flat-list-mvcp": ^0.10.3 - "@stream-io/react-native-webrtc": 118.1.0 + "@stream-io/react-native-webrtc": 124.0.0-rc.1 "@stream-io/video-filters-react-native": "workspace:^" "@stream-io/video-react-native-sdk": "workspace:^" "@types/eslint": ^8.56.10 From c4e828ee15bf618782ac48936f2978d8b65ff61b Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 15 Oct 2024 17:17:52 +0200 Subject: [PATCH 22/33] feat: optimal codec selection, refactored Publisher for improved readability --- packages/client/src/rtc/Publisher.ts | 225 ++++++++---------- .../src/rtc/__tests__/Publisher.test.ts | 8 +- .../src/rtc/__tests__/videoLayers.test.ts | 32 +-- packages/client/src/rtc/bitrateLookup.ts | 4 +- packages/client/src/rtc/codecs.ts | 38 ++- packages/client/src/rtc/videoLayers.ts | 35 ++- packages/client/src/types.ts | 6 +- .../react-dogfood/components/MeetingUI.tsx | 5 +- 8 files changed, 178 insertions(+), 175 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index cb180dcae7..8b881c9e7d 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -12,6 +12,7 @@ import { findOptimalScreenSharingLayers, findOptimalVideoLayers, OptimalVideoLayer, + toSvcEncodings, } from './videoLayers'; import { getOptimalVideoCodec, getPreferredCodecs, isSvcCodec } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; @@ -57,17 +58,6 @@ export class Publisher { * @internal */ private readonly transceiverInitOrder: TrackType[] = []; - - private readonly trackKindMapping: { - [key in TrackType]: 'video' | 'audio' | undefined; - } = { - [TrackType.AUDIO]: 'audio', - [TrackType.VIDEO]: 'video', - [TrackType.SCREEN_SHARE]: 'video', - [TrackType.SCREEN_SHARE_AUDIO]: 'audio', - [TrackType.UNSPECIFIED]: undefined, - }; - private readonly isDtxEnabled: boolean; private readonly isRedEnabled: boolean; @@ -216,111 +206,96 @@ export class Publisher { throw new Error(`Can't publish a track that has ended already.`); } - let transceiver = this.pc - .getTransceivers() - .find( - (t) => - t === this.transceiverCache.get(trackType) && - t.sender.track && - t.sender.track?.kind === this.trackKindMapping[trackType], - ); - - /** - * An event handler which listens for the 'ended' event on the track. - * Once the track has ended, it will notify the SFU and update the state. - */ - const handleTrackEnded = () => { - this.logger( - 'info', - `Track ${TrackType[trackType]} has ended abruptly, notifying the SFU`, - ); - // cleanup, this event listener needs to run only once. - track.removeEventListener('ended', handleTrackEnded); - this.notifyTrackMuteStateChanged(mediaStream, trackType, true).catch( - (err) => this.logger('warn', `Couldn't notify track mute state`, err), - ); - }; - - if (!transceiver) { - const { settings } = this.state; - const targetResolution = settings?.video - .target_resolution as TargetResolutionResponse; - const screenShareBitrate = - settings?.screensharing.target_resolution?.bitrate; - - const videoEncodings = - trackType === TrackType.VIDEO - ? findOptimalVideoLayers(track, targetResolution, opts) - : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers(track, opts, screenShareBitrate) - : undefined; + // enable the track if it is disabled + if (!track.enabled) track.enabled = true; + const transceiver = this.transceiverCache.get(trackType); + if (!transceiver || !transceiver.sender.track) { // listen for 'ended' event on the track as it might be ended abruptly - // by an external factor as permission revokes, device disconnected, etc. + // by an external factors such as permission revokes, a disconnected device, etc. // keep in mind that `track.stop()` doesn't trigger this event. - track.addEventListener('ended', handleTrackEnded); - if (!track.enabled) { - track.enabled = true; - } - - const { preferredCodec } = opts; - const usesSvcCodec = isSvcCodec(preferredCodec); - transceiver = this.pc.addTransceiver(track, { - direction: 'sendonly', - streams: - trackType === TrackType.VIDEO || trackType === TrackType.SCREEN_SHARE - ? [mediaStream] - : undefined, - sendEncodings: usesSvcCodec - ? videoEncodings - ?.filter((l) => l.rid === 'f') - .map((l) => ({ ...l, rid: 'q' })) // announce the 'f' layer as 'q' - : videoEncodings, - }); - - this.logger('debug', `Added ${TrackType[trackType]} transceiver`); - this.transceiverInitOrder.push(trackType); - this.transceiverCache.set(trackType, transceiver); - this.publishOptsForTrack.set(trackType, opts); - - const codec = - trackType === TrackType.VIDEO && !preferredCodec - ? getOptimalVideoCodec() - : preferredCodec; - - const codecPreferences = - 'setCodecPreferences' in transceiver - ? this.getCodecPreferences(trackType, codec) - : undefined; - if (codecPreferences) { - this.logger( - 'info', - `Setting ${TrackType[trackType]} codec preferences`, - codecPreferences, + const handleTrackEnded = () => { + this.logger('info', `Track ${TrackType[trackType]} has ended abruptly`); + track.removeEventListener('ended', handleTrackEnded); + this.notifyTrackMuteStateChanged(mediaStream, trackType, true).catch( + (err) => this.logger('warn', `Couldn't notify track mute state`, err), ); - try { - transceiver.setCodecPreferences(codecPreferences); - } catch (err) { - this.logger('warn', `Couldn't set codec preferences`, err); - } - } + }; + track.addEventListener('ended', handleTrackEnded); + this.addTransceiver(trackType, track, opts, mediaStream); } else { - const previousTrack = transceiver.sender.track; - // don't stop the track if we are re-publishing the same track - if (previousTrack && previousTrack !== track) { - previousTrack.stop(); - previousTrack.removeEventListener('ended', handleTrackEnded); - track.addEventListener('ended', handleTrackEnded); - } - if (!track.enabled) { - track.enabled = true; - } - await transceiver.sender.replaceTrack(track); + await this.updateTransceiver(transceiver, track); } await this.notifyTrackMuteStateChanged(mediaStream, trackType, false); }; + /** + * Adds a new transceiver to the peer connection. + * This needs to be called when a new track kind is added to the peer connection. + * In other cases, use `updateTransceiver` method. + */ + private addTransceiver = ( + trackType: TrackType, + track: MediaStreamTrack, + opts: PublishOptions, + mediaStream: MediaStream, + ) => { + const codecInUse = getOptimalVideoCodec(opts.preferredCodec); + const videoEncodings = this.computeLayers(trackType, track, opts); + const transceiver = this.pc.addTransceiver(track, { + direction: 'sendonly', + streams: + trackType === TrackType.VIDEO || trackType === TrackType.SCREEN_SHARE + ? [mediaStream] + : undefined, + sendEncodings: isSvcCodec(codecInUse) + ? toSvcEncodings(videoEncodings) + : videoEncodings, + }); + + this.logger('debug', `Added ${TrackType[trackType]} transceiver`); + this.transceiverInitOrder.push(trackType); + this.transceiverCache.set(trackType, transceiver); + this.publishOptsForTrack.set(trackType, opts); + + // handle codec preferences + if (!('setCodecPreferences' in transceiver)) return; + + const codecPreferences = this.getCodecPreferences( + trackType, + trackType === TrackType.VIDEO ? codecInUse : undefined, + ); + if (!codecPreferences) return; + + try { + this.logger( + 'info', + `Setting ${TrackType[trackType]} codec preferences`, + codecPreferences, + ); + transceiver.setCodecPreferences(codecPreferences); + } catch (err) { + this.logger('warn', `Couldn't set codec preferences`, err); + } + }; + + /** + * Updates the given transceiver with the new track. + * Stops the previous track and replaces it with the new one. + */ + private updateTransceiver = async ( + transceiver: RTCRtpTransceiver, + track: MediaStreamTrack, + ) => { + const previousTrack = transceiver.sender.track; + // don't stop the track if we are re-publishing the same track + if (previousTrack && previousTrack !== track) { + previousTrack.stop(); + } + await transceiver.sender.replaceTrack(track); + }; + /** * Stops publishing the given track type to the SFU, if it is currently being published. * Underlying track will be stopped and removed from the publisher. @@ -328,11 +303,7 @@ export class Publisher { * @param stopTrack specifies whether track should be stopped or just disabled */ unpublishStream = async (trackType: TrackType, stopTrack: boolean) => { - const transceiver = this.pc - .getTransceivers() - .find( - (t) => t === this.transceiverCache.get(trackType) && t.sender.track, - ); + const transceiver = this.transceiverCache.get(trackType); if ( transceiver && transceiver.sender.track && @@ -669,10 +640,6 @@ export class Publisher { */ getAnnouncedTracks = (sdp?: string): TrackInfo[] => { sdp = sdp || this.pc.localDescription?.sdp; - - const { settings } = this.state; - const targetResolution = settings?.video - .target_resolution as TargetResolutionResponse; return this.pc .getTransceivers() .filter((t) => t.direction === 'sendonly' && t.sender.track) @@ -685,13 +652,7 @@ export class Publisher { let optimalLayers: OptimalVideoLayer[]; const isTrackLive = track.readyState === 'live'; if (isTrackLive) { - const publishOpts = this.publishOptsForTrack.get(trackType); - optimalLayers = - trackType === TrackType.VIDEO - ? findOptimalVideoLayers(track, targetResolution, publishOpts) - : trackType === TrackType.SCREEN_SHARE - ? findOptimalScreenSharingLayers(track, publishOpts) - : []; + optimalLayers = this.computeLayers(trackType, track) || []; this.trackLayersCache.set(trackType, optimalLayers); } else { // we report the last known optimal layers for ended tracks @@ -736,6 +697,26 @@ export class Publisher { }); }; + private computeLayers = ( + trackType: TrackType, + track: MediaStreamTrack, + opts?: PublishOptions, + ): OptimalVideoLayer[] | undefined => { + const { settings } = this.state; + const targetResolution = settings?.video + .target_resolution as TargetResolutionResponse; + const screenShareBitrate = + settings?.screensharing.target_resolution?.bitrate; + + const publishOpts = opts || this.publishOptsForTrack.get(trackType); + const codecInUse = getOptimalVideoCodec(publishOpts?.preferredCodec); + return trackType === TrackType.VIDEO + ? findOptimalVideoLayers(track, targetResolution, codecInUse, publishOpts) + : trackType === TrackType.SCREEN_SHARE + ? findOptimalScreenSharingLayers(track, publishOpts, screenShareBitrate) + : undefined; + }; + private onIceCandidateError = (e: Event) => { const errorMessage = e instanceof RTCPeerConnectionIceErrorEvent && diff --git a/packages/client/src/rtc/__tests__/Publisher.test.ts b/packages/client/src/rtc/__tests__/Publisher.test.ts index c8d573a014..9b73af5b95 100644 --- a/packages/client/src/rtc/__tests__/Publisher.test.ts +++ b/packages/client/src/rtc/__tests__/Publisher.test.ts @@ -139,11 +139,7 @@ describe('Publisher', () => { vi.spyOn(transceiver.sender, 'track', 'get').mockReturnValue(newTrack); expect(track.stop).toHaveBeenCalled(); - expect(track.removeEventListener).toHaveBeenCalledWith( - 'ended', - expect.any(Function), - ); - expect(newTrack.addEventListener).toHaveBeenCalledWith( + expect(newTrack.addEventListener).not.toHaveBeenCalledWith( 'ended', expect.any(Function), ); @@ -157,7 +153,7 @@ describe('Publisher', () => { ); }); - it('can publish and un-pubish with just enabling and disabling tracks', async () => { + it('can publish and un-publish with just enabling and disabling tracks', async () => { const mediaStream = new MediaStream(); const track = new MediaStreamTrack(); mediaStream.addTrack(track); diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index 593c9c5cb3..67a84cb7fb 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -141,7 +141,7 @@ describe('videoLayers', () => { width: 1280, height: 720, }); - const layers = findOptimalVideoLayers(track, undefined, { + const layers = findOptimalVideoLayers(track, undefined, 'vp9', { preferredCodec: 'vp9', scalabilityMode: 'L3T3', }); @@ -160,6 +160,7 @@ describe('videoLayers', () => { 1280, 720, undefined, + undefined, ); expect(scaledBitrate).toBe(1333333); }); @@ -175,6 +176,7 @@ describe('videoLayers', () => { width, height, undefined, + undefined, ); downscaleFactor *= 2; return { @@ -193,45 +195,25 @@ describe('videoLayers', () => { it('should not scale target bitrate if resolution is larger than target resolution', () => { const targetResolution = { width: 1280, height: 720, bitrate: 1000000 }; - const scaledBitrate = getComputedMaxBitrate( - targetResolution, - 2560, - 1440, - undefined, - ); + const scaledBitrate = getComputedMaxBitrate(targetResolution, 2560, 1440); expect(scaledBitrate).toBe(1000000); }); it('should not scale target bitrate if resolution is equal to target resolution', () => { const targetResolution = { width: 1280, height: 720, bitrate: 1000000 }; - const scaledBitrate = getComputedMaxBitrate( - targetResolution, - 1280, - 720, - undefined, - ); + const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720); expect(scaledBitrate).toBe(1000000); }); it('should handle 0 width and height', () => { const targetResolution = { width: 1280, height: 720, bitrate: 1000000 }; - const scaledBitrate = getComputedMaxBitrate( - targetResolution, - 0, - 0, - undefined, - ); + const scaledBitrate = getComputedMaxBitrate(targetResolution, 0, 0); expect(scaledBitrate).toBe(0); }); it('should handle 4k target resolution', () => { const targetResolution = { width: 3840, height: 2160, bitrate: 15000000 }; - const scaledBitrate = getComputedMaxBitrate( - targetResolution, - 1280, - 720, - undefined, - ); + const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720); expect(scaledBitrate).toBe(1666667); }); }); diff --git a/packages/client/src/rtc/bitrateLookup.ts b/packages/client/src/rtc/bitrateLookup.ts index 8d961816f5..70fcf6f6bd 100644 --- a/packages/client/src/rtc/bitrateLookup.ts +++ b/packages/client/src/rtc/bitrateLookup.ts @@ -43,10 +43,10 @@ const bitrateLookupTable: Record< }; export const getOptimalBitrate = ( - codec: string, + codec: PreferredCodec, frameHeight: number, ): number => { - const codecLookup = bitrateLookupTable[codec.toLowerCase()]; + const codecLookup = bitrateLookupTable[codec]; if (!codecLookup) throw new Error(`Unknown codec: ${codec}`); let bitrate = codecLookup[frameHeight]; diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index c7ad0969ef..7a7899189b 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -97,16 +97,42 @@ export const getGenericSdp = async (direction: RTCRtpTransceiverDirection) => { /** * Returns the optimal video codec for the device. */ -export const getOptimalVideoCodec = (): PreferredCodec => { +export const getOptimalVideoCodec = ( + preferredCodec: PreferredCodec | undefined, +): PreferredCodec => { if (isReactNative()) { - const osName = getOSInfo()?.name.toLowerCase(); - if (osName === 'ios' || osName === 'ipados') return 'h264'; - if (osName === 'android') return 'vp8'; // TODO switch to vp9 - return 'vp8'; + const os = getOSInfo()?.name.toLowerCase(); + if (os === 'android') return preferredOr(preferredCodec, 'vp9'); + if (os === 'ios' || os === 'ipados') return 'h264'; + return preferredOr(preferredCodec, 'h264'); } if (isSafari()) return 'h264'; if (isFirefox()) return 'vp8'; - return 'vp8'; // TODO switch to vp9 + return preferredOr(preferredCodec, 'vp8'); +}; + +/** + * Determines if the platform supports the preferred codec. + * If not, it returns the fallback codec. + */ +const preferredOr = ( + codec: PreferredCodec | undefined, + fallback: PreferredCodec, +): PreferredCodec => { + if (!codec) return fallback; + if (!('getCapabilities' in RTCRtpSender)) return fallback; + const capabilities = RTCRtpSender.getCapabilities('video'); + if (!capabilities) return fallback; + + // Safari and Firefox do not have a good support encoding to SVC codecs, + // so we disable it for them. + if (isSvcCodec(codec) && (isSafari() || isFirefox())) return fallback; + + const { codecs } = capabilities; + const codecMimeType = `video/${codec}`.toLowerCase(); + return codecs.some((c) => c.mimeType.toLowerCase() === codecMimeType) + ? codec + : fallback; }; /** diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index 9ac9bd90d3..c739d71b1b 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -1,4 +1,4 @@ -import { PublishOptions } from '../types'; +import { PreferredCodec, PublishOptions } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; import { isSvcCodec } from './codecs'; import { getOptimalBitrate } from './bitrateLookup'; @@ -23,36 +23,47 @@ const defaultBitratePerRid: Record = { f: DEFAULT_BITRATE, }; +/** + * In SVC, we need to send only one video encoding (layer). + * this layer will have the additional spatial and temporal layers + * defined via the scalabilityMode property. + * + * @param layers the layers to process. + */ +export const toSvcEncodings = (layers: OptimalVideoLayer[] | undefined) => { + // we take the `f` layer, and we rename it to `q`. + return layers?.filter((l) => l.rid === 'f').map((l) => ({ ...l, rid: 'q' })); +}; + /** * Determines the most optimal video layers for simulcasting * for the given track. * * @param videoTrack the video track to find optimal layers for. * @param targetResolution the expected target resolution. + * @param codecInUse the codec in use. * @param publishOptions the publish options for the track. */ export const findOptimalVideoLayers = ( videoTrack: MediaStreamTrack, targetResolution: TargetResolutionResponse = defaultTargetResolution, + codecInUse?: PreferredCodec, publishOptions?: PublishOptions, ) => { const optimalVideoLayers: OptimalVideoLayer[] = []; const settings = videoTrack.getSettings(); const { width = 0, height = 0 } = settings; - const { - preferredCodec, - scalabilityMode, - bitrateDownscaleFactor = 2, - } = publishOptions || {}; + const { scalabilityMode, bitrateDownscaleFactor = 2 } = publishOptions || {}; const maxBitrate = getComputedMaxBitrate( targetResolution, width, height, + codecInUse, publishOptions, ); let downscaleFactor = 1; let bitrateFactor = 1; - const svcCodec = isSvcCodec(preferredCodec); + const svcCodec = isSvcCodec(codecInUse); for (const rid of ['f', 'h', 'q']) { const layer: OptimalVideoLayer = { active: true, @@ -98,13 +109,15 @@ export const findOptimalVideoLayers = ( * @param targetResolution the target resolution. * @param currentWidth the current width of the track. * @param currentHeight the current height of the track. + * @param codecInUse the codec in use. * @param publishOptions the publish options. */ export const getComputedMaxBitrate = ( targetResolution: TargetResolutionResponse, currentWidth: number, currentHeight: number, - publishOptions: PublishOptions | undefined, + codecInUse?: PreferredCodec, + publishOptions?: PublishOptions, ): number => { // if the current resolution is lower than the target resolution, // we want to proportionally reduce the target bitrate @@ -113,14 +126,12 @@ export const getComputedMaxBitrate = ( height: targetHeight, bitrate: targetBitrate, } = targetResolution; - const { preferredBitrate, preferredCodec } = publishOptions || {}; + const { preferredBitrate } = publishOptions || {}; const frameHeight = currentWidth > currentHeight ? currentHeight : currentWidth; const bitrate = preferredBitrate || - (preferredCodec - ? getOptimalBitrate(preferredCodec, frameHeight) - : targetBitrate); + (codecInUse ? getOptimalBitrate(codecInUse, frameHeight) : targetBitrate); if (currentWidth < targetWidth || currentHeight < targetHeight) { const currentPixels = currentWidth * currentHeight; const targetPixels = targetWidth * targetHeight; diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 546e9872a4..024e7b6e27 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -150,8 +150,12 @@ export type SubscriptionChanges = { * A preferred codec to use when publishing a video track. * @internal */ -export type PreferredCodec = 'vp8' | 'h264' | 'vp9' | 'av1' | string; +export type PreferredCodec = 'vp8' | 'h264' | 'vp9' | 'av1'; +/** + * A collection of track publication options. + * @internal + */ export type PublishOptions = { /** * The preferred codec to use when publishing the video stream. diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index b46c081f17..1d066bdacb 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -7,6 +7,7 @@ import { defaultSortPreset, LoadingIndicator, noopComparator, + PreferredCodec, useCall, useCallStateHooks, usePersistedDevicePreferences, @@ -48,7 +49,9 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const callState = useCallCallingState(); const isProntoEnvironment = useIsProntoEnvironment(); - const videoCodecOverride = router.query['video_codec'] as string | undefined; + const videoCodecOverride = router.query['video_codec'] as + | PreferredCodec + | undefined; const bitrateOverride = router.query['bitrate'] as string | undefined; const bitrateFactorOverride = router.query['bitrate_factor'] as | string From 696a4ef4f13d94dac900a968cc481df34b9d5854 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 15 Oct 2024 18:03:56 +0200 Subject: [PATCH 23/33] chore: moved `updatePublishOptions` to the `call` instance --- packages/client/src/Call.ts | 44 +++++++++---------- packages/client/src/devices/CameraManager.ts | 24 ++-------- .../client/src/devices/ScreenShareManager.ts | 4 +- .../dogfood/src/components/MeetingUI.tsx | 3 ++ .../react-dogfood/components/MeetingUI.tsx | 9 +--- 5 files changed, 30 insertions(+), 54 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 8a9a59e091..cbf052dd10 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -199,6 +199,7 @@ export class Call { */ private readonly dispatcher = new Dispatcher(); + private publishOptions?: PublishOptions; private statsReporter?: StatsReporter; private sfuStatsReporter?: SfuStatsReporter; private dropTimeout: ReturnType | undefined; @@ -1289,19 +1290,12 @@ export class Call { break; case TrackType.VIDEO: const videoStream = this.camera.state.mediaStream; - if (videoStream) { - await this.publishVideoStream( - videoStream, - this.camera.publishOptions, - ); - } + if (videoStream) await this.publishVideoStream(videoStream); break; case TrackType.SCREEN_SHARE: const screenShareStream = this.screenShare.state.mediaStream; if (screenShareStream) { - await this.publishScreenShareStream(screenShareStream, { - screenShareSettings: this.screenShare.getSettings(), - }); + await this.publishScreenShareStream(screenShareStream); } break; // screen share audio can't exist without a screen share, so we handle it there @@ -1333,12 +1327,8 @@ export class Call { * The previous video stream will be stopped. * * @param videoStream the video stream to publish. - * @param opts the options to use when publishing the stream. */ - publishVideoStream = async ( - videoStream: MediaStream, - opts: PublishOptions = {}, - ) => { + publishVideoStream = async (videoStream: MediaStream) => { if (!this.sfuClient) throw new Error(`Call not joined yet.`); // joining is in progress, and we should wait until the client is ready await this.sfuClient.joinTask; @@ -1360,7 +1350,7 @@ export class Call { videoStream, videoTrack, TrackType.VIDEO, - opts, + this.publishOptions, ); }; @@ -1404,12 +1394,8 @@ export class Call { * The previous screen-share stream will be stopped. * * @param screenShareStream the screen-share stream to publish. - * @param opts the options to use when publishing the stream. */ - publishScreenShareStream = async ( - screenShareStream: MediaStream, - opts: PublishOptions = {}, - ) => { + publishScreenShareStream = async (screenShareStream: MediaStream) => { if (!this.sfuClient) throw new Error(`Call not joined yet.`); // joining is in progress, and we should wait until the client is ready await this.sfuClient.joinTask; @@ -1428,6 +1414,9 @@ export class Call { if (!this.trackPublishOrder.includes(TrackType.SCREEN_SHARE)) { this.trackPublishOrder.push(TrackType.SCREEN_SHARE); } + const opts: PublishOptions = { + screenShareSettings: this.screenShare.getSettings(), + }; await this.publisher.publishStream( screenShareStream, screenShareTrack, @@ -1464,6 +1453,16 @@ export class Call { await this.publisher?.unpublishStream(trackType, stopTrack); }; + /** + * Updates the preferred publishing options + * + * @internal + * @param options the options to use. + */ + updatePublishOptions(options: PublishOptions) { + this.publishOptions = { ...this.publishOptions, ...options }; + } + /** * Notifies the SFU that a noise cancellation process has started. * @@ -2085,10 +2084,7 @@ export class Call { this.camera.state.mediaStream && !this.publisher?.isPublishing(TrackType.VIDEO) ) { - await this.publishVideoStream( - this.camera.state.mediaStream, - this.camera.publishOptions, - ); + await this.publishVideoStream(this.camera.state.mediaStream); } // Start camera if backend config specifies, and there is no local setting diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index 259f3bf59e..847bba9437 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -4,7 +4,7 @@ import { CameraDirection, CameraManagerState } from './CameraManagerState'; import { InputMediaDeviceManager } from './InputMediaDeviceManager'; import { getVideoDevices, getVideoStream } from './devices'; import { TrackType } from '../gen/video/sfu/models/models'; -import { PreferredCodec, PublishOptions } from '../types'; +import { PreferredCodec } from '../types'; export class CameraManager extends InputMediaDeviceManager { private targetResolution = { @@ -12,13 +12,6 @@ export class CameraManager extends InputMediaDeviceManager { height: 720, }; - /** - * The options to use when publishing the video stream. - * - * @internal - */ - publishOptions: PublishOptions | undefined; - /** * Constructs a new CameraManager. * @@ -86,20 +79,11 @@ export class CameraManager extends InputMediaDeviceManager { * Sets the preferred codec for encoding the video. * * @internal internal use only, not part of the public API. + * @deprecated use {@link call.updatePublishOptions} instead. * @param codec the codec to use for encoding the video. */ setPreferredCodec(codec: PreferredCodec | undefined) { - this.updatePublishOptions({ preferredCodec: codec }); - } - - /** - * Updates the preferred publish options for the video stream. - * - * @internal - * @param options the options to use. - */ - updatePublishOptions(options: PublishOptions) { - this.publishOptions = { ...this.publishOptions, ...options }; + this.call.updatePublishOptions({ preferredCodec: codec }); } protected getDevices(): Observable { @@ -121,7 +105,7 @@ export class CameraManager extends InputMediaDeviceManager { } protected publishStream(stream: MediaStream): Promise { - return this.call.publishVideoStream(stream, this.publishOptions); + return this.call.publishVideoStream(stream); } protected stopPublishStream(stopTracks: boolean): Promise { diff --git a/packages/client/src/devices/ScreenShareManager.ts b/packages/client/src/devices/ScreenShareManager.ts index f9404d18dc..b71b617bf5 100644 --- a/packages/client/src/devices/ScreenShareManager.ts +++ b/packages/client/src/devices/ScreenShareManager.ts @@ -80,9 +80,7 @@ export class ScreenShareManager extends InputMediaDeviceManager< } protected publishStream(stream: MediaStream): Promise { - return this.call.publishScreenShareStream(stream, { - screenShareSettings: this.state.settings, - }); + return this.call.publishScreenShareStream(stream); } protected async stopPublishStream(stopTracks: boolean): Promise { diff --git a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx index dcae405b3e..378270ab03 100644 --- a/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx +++ b/sample-apps/react-native/dogfood/src/components/MeetingUI.tsx @@ -59,6 +59,9 @@ export const MeetingUI = ({ callId, navigation, route }: Props) => { const onJoinCallHandler = useCallback(async () => { try { setShow('loading'); + call?.updatePublishOptions({ + preferredCodec: 'vp9', + }); await call?.join({ create: true }); appStoreSetState({ chatLabelNoted: false }); setShow('active-call'); diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 1d066bdacb..936389fe5d 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -1,6 +1,5 @@ import { useRouter } from 'next/router'; import { JSX, useCallback, useEffect, useState } from 'react'; -import { isFirefox } from 'mobile-device-detect'; import Gleap from 'gleap'; import { CallingState, @@ -23,7 +22,6 @@ import { import { ActiveCall } from './ActiveCall'; import { Feedback } from './Feedback/Feedback'; import { DefaultAppHeader } from './DefaultAppHeader'; -import { useIsProntoEnvironment } from '../context/AppEnvironmentContext'; const contents = { 'error-join': { @@ -48,7 +46,6 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { const { useCallCallingState } = useCallStateHooks(); const callState = useCallCallingState(); - const isProntoEnvironment = useIsProntoEnvironment(); const videoCodecOverride = router.query['video_codec'] as | PreferredCodec | undefined; @@ -65,14 +62,13 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { if (!fastJoin) setShow('loading'); if (!call) throw new Error('No active call found'); try { - const prontoDefaultCodec = - isProntoEnvironment && !isFirefox ? 'vp9' : 'vp8'; + const prontoDefaultCodec = 'vp9'; const preferredCodec = videoCodecOverride || prontoDefaultCodec; const preferredBitrate = bitrateOverride ? parseInt(bitrateOverride, 10) : undefined; - call.camera.updatePublishOptions({ + call.updatePublishOptions({ preferredCodec, scalabilityMode, preferredBitrate, @@ -93,7 +89,6 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { bitrateFactorOverride, bitrateOverride, call, - isProntoEnvironment, scalabilityMode, videoCodecOverride, ], From 7071ad765b96ae63b416c395a6ce14becbb65ed1 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 15 Oct 2024 18:20:13 +0200 Subject: [PATCH 24/33] feat: add `forceCodec` option --- packages/client/src/rtc/Publisher.ts | 3 ++- packages/client/src/types.ts | 6 ++++++ sample-apps/react/react-dogfood/components/MeetingUI.tsx | 5 ++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 8b881c9e7d..09a9dd0c76 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -241,7 +241,8 @@ export class Publisher { opts: PublishOptions, mediaStream: MediaStream, ) => { - const codecInUse = getOptimalVideoCodec(opts.preferredCodec); + const { forceCodec, preferredCodec } = opts; + const codecInUse = forceCodec || getOptimalVideoCodec(preferredCodec); const videoEncodings = this.computeLayers(trackType, track, opts); const transceiver = this.pc.addTransceiver(track, { direction: 'sendonly', diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 024e7b6e27..e32c82694e 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -161,6 +161,12 @@ export type PublishOptions = { * The preferred codec to use when publishing the video stream. */ preferredCodec?: PreferredCodec; + /** + * Force the codec to use when publishing the video stream. + * This will override the preferred codec and the internal codec selection logic. + * Use with caution. + */ + forceCodec?: PreferredCodec; /** * The preferred scalability to use when publishing the video stream. * Applicable only for SVC codecs. diff --git a/sample-apps/react/react-dogfood/components/MeetingUI.tsx b/sample-apps/react/react-dogfood/components/MeetingUI.tsx index 936389fe5d..5479dae277 100644 --- a/sample-apps/react/react-dogfood/components/MeetingUI.tsx +++ b/sample-apps/react/react-dogfood/components/MeetingUI.tsx @@ -62,14 +62,13 @@ export const MeetingUI = ({ chatClient, mode }: MeetingUIProps) => { if (!fastJoin) setShow('loading'); if (!call) throw new Error('No active call found'); try { - const prontoDefaultCodec = 'vp9'; - const preferredCodec = videoCodecOverride || prontoDefaultCodec; const preferredBitrate = bitrateOverride ? parseInt(bitrateOverride, 10) : undefined; call.updatePublishOptions({ - preferredCodec, + preferredCodec: 'vp9', + forceCodec: videoCodecOverride, scalabilityMode, preferredBitrate, bitrateDownscaleFactor: bitrateFactorOverride From 850301bae9577b457b8d2b89d24695c3b6e1e5f7 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Tue, 15 Oct 2024 18:23:07 +0200 Subject: [PATCH 25/33] chore: adjust tests --- .../src/devices/__tests__/CameraManager.test.ts | 15 --------------- .../devices/__tests__/ScreenShareManager.test.ts | 14 -------------- 2 files changed, 29 deletions(-) diff --git a/packages/client/src/devices/__tests__/CameraManager.test.ts b/packages/client/src/devices/__tests__/CameraManager.test.ts index 656b5e48b2..a1b32a7f74 100644 --- a/packages/client/src/devices/__tests__/CameraManager.test.ts +++ b/packages/client/src/devices/__tests__/CameraManager.test.ts @@ -82,21 +82,6 @@ describe('CameraManager', () => { expect(manager['call'].publishVideoStream).toHaveBeenCalledWith( manager.state.mediaStream, - undefined, - ); - }); - - it('publish stream with preferred codec', async () => { - manager['call'].state.setCallingState(CallingState.JOINED); - manager.setPreferredCodec('h264'); - - await manager.enable(); - - expect(manager['call'].publishVideoStream).toHaveBeenCalledWith( - manager.state.mediaStream, - { - preferredCodec: 'h264', - }, ); }); diff --git a/packages/client/src/devices/__tests__/ScreenShareManager.test.ts b/packages/client/src/devices/__tests__/ScreenShareManager.test.ts index c011cc20b9..8c156298ca 100644 --- a/packages/client/src/devices/__tests__/ScreenShareManager.test.ts +++ b/packages/client/src/devices/__tests__/ScreenShareManager.test.ts @@ -115,20 +115,6 @@ describe('ScreenShareManager', () => { await manager.enable(); expect(call.publishScreenShareStream).toHaveBeenCalledWith( manager.state.mediaStream, - { screenShareSettings: undefined }, - ); - }); - - it('publishes screen share stream with settings', async () => { - const call = manager['call']; - call.state.setCallingState(CallingState.JOINED); - - manager.setSettings({ maxFramerate: 15, maxBitrate: 1000 }); - - await manager.enable(); - expect(call.publishScreenShareStream).toHaveBeenCalledWith( - manager.state.mediaStream, - { screenShareSettings: { maxFramerate: 15, maxBitrate: 1000 } }, ); }); From fd4c0c5706bf4d0df1ada65d973e4273521f9cc8 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 12:13:55 +0200 Subject: [PATCH 26/33] chore: reorganize Publisher --- packages/client/src/helpers/sdp-munging.ts | 53 +++++--- packages/client/src/rtc/Publisher.ts | 113 ++++-------------- .../src/rtc/__tests__/videoLayers.test.ts | 44 ++++--- packages/client/src/rtc/videoLayers.ts | 12 ++ 4 files changed, 103 insertions(+), 119 deletions(-) diff --git a/packages/client/src/helpers/sdp-munging.ts b/packages/client/src/helpers/sdp-munging.ts index 0b2df45d1e..b8bdfd66af 100644 --- a/packages/client/src/helpers/sdp-munging.ts +++ b/packages/client/src/helpers/sdp-munging.ts @@ -118,21 +118,15 @@ const getOpusFmtp = (sdp: string): Fmtp | undefined => { */ export const toggleDtx = (sdp: string, enable: boolean): string => { const opusFmtp = getOpusFmtp(sdp); - if (opusFmtp) { - const matchDtx = /usedtx=(\d)/.exec(opusFmtp.config); - const requiredDtxConfig = `usedtx=${enable ? '1' : '0'}`; - if (matchDtx) { - const newFmtp = opusFmtp.original.replace( - /usedtx=(\d)/, - requiredDtxConfig, - ); - return sdp.replace(opusFmtp.original, newFmtp); - } else { - const newFmtp = `${opusFmtp.original};${requiredDtxConfig}`; - return sdp.replace(opusFmtp.original, newFmtp); - } - } - return sdp; + if (!opusFmtp) return sdp; + + const matchDtx = /usedtx=(\d)/.exec(opusFmtp.config); + const requiredDtxConfig = `usedtx=${enable ? '1' : '0'}`; + const newFmtp = matchDtx + ? opusFmtp.original.replace(/usedtx=(\d)/, requiredDtxConfig) + : `${opusFmtp.original};${requiredDtxConfig}`; + + return sdp.replace(opusFmtp.original, newFmtp); }; /** @@ -181,3 +175,32 @@ export const enableHighQualityAudio = ( return SDP.write(parsedSdp); }; + +/** + * Extracts the mid from the transceiver or the SDP. + * + * @param transceiver the transceiver. + * @param transceiverInitIndex the index of the transceiver in the transceiver's init array. + * @param sdp the SDP. + */ +export const extractMid = ( + transceiver: RTCRtpTransceiver, + transceiverInitIndex: number, + sdp: string | undefined, +): string => { + if (transceiver.mid) return transceiver.mid; + if (!sdp) return ''; + + const track = transceiver.sender.track!; + const parsedSdp = SDP.parse(sdp); + const media = parsedSdp.media.find((m) => { + return ( + m.type === track.kind && + // if `msid` is not present, we assume that the track is the first one + (m.msid?.includes(track.id) ?? true) + ); + }); + if (typeof media?.mid !== 'undefined') return String(media.mid); + if (transceiverInitIndex === -1) return ''; + return String(transceiverInitIndex); +}; diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts index 09a9dd0c76..a55de6411e 100644 --- a/packages/client/src/rtc/Publisher.ts +++ b/packages/client/src/rtc/Publisher.ts @@ -1,24 +1,27 @@ -import * as SDP from 'sdp-transform'; import { StreamSfuClient } from '../StreamSfuClient'; import { PeerType, TrackInfo, TrackType, VideoLayer, - VideoQuality, } from '../gen/video/sfu/models/models'; import { getIceCandidate } from './helpers/iceCandidate'; import { findOptimalScreenSharingLayers, findOptimalVideoLayers, OptimalVideoLayer, + ridToVideoQuality, toSvcEncodings, } from './videoLayers'; import { getOptimalVideoCodec, getPreferredCodecs, isSvcCodec } from './codecs'; import { trackTypeToParticipantStreamKey } from './helpers/tracks'; import { CallingState, CallState } from '../store'; import { PublishOptions } from '../types'; -import { enableHighQualityAudio, toggleDtx } from '../helpers/sdp-munging'; +import { + enableHighQualityAudio, + extractMid, + toggleDtx, +} from '../helpers/sdp-munging'; import { Logger } from '../coordinator/connection/types'; import { getLogger } from '../logger'; import { Dispatcher } from './Dispatcher'; @@ -66,24 +69,10 @@ export class Publisher { private readonly onUnrecoverableError?: () => void; private isIceRestarting = false; - - /** - * The SFU client instance to use for publishing and signaling. - */ - sfuClient: StreamSfuClient; + private sfuClient: StreamSfuClient; /** * Constructs a new `Publisher` instance. - * - * @param connectionConfig the connection configuration to use. - * @param sfuClient the SFU client to use. - * @param state the call state to use. - * @param dispatcher the dispatcher to use. - * @param isDtxEnabled whether DTX is enabled. - * @param isRedEnabled whether RED is enabled. - * @param iceRestartDelay the delay in milliseconds to wait before restarting ICE once connection goes to `disconnected` state. - * @param onUnrecoverableError a callback to call when an unrecoverable error occurs. - * @param logTag the log tag to use. */ constructor({ connectionConfig, @@ -535,23 +524,22 @@ export class Publisher { */ private negotiate = async (options?: RTCOfferOptions) => { const offer = await this.pc.createOffer(options); - let sdp = this.mungeCodecs(offer.sdp); - if (sdp && this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { - sdp = this.enableHighQualityAudio(sdp); + if (offer.sdp) { + offer.sdp = toggleDtx(offer.sdp, this.isDtxEnabled); + if (this.isPublishing(TrackType.SCREEN_SHARE_AUDIO)) { + offer.sdp = this.enableHighQualityAudio(offer.sdp); + } } - // set the munged SDP back to the offer - offer.sdp = sdp; - const trackInfos = this.getAnnouncedTracks(offer.sdp); if (trackInfos.length === 0) { throw new Error(`Can't negotiate without announcing any tracks`); } - this.isIceRestarting = options?.iceRestart ?? false; - await this.pc.setLocalDescription(offer); - try { + this.isIceRestarting = options?.iceRestart ?? false; + await this.pc.setLocalDescription(offer); + const { response } = await this.sfuClient.setPublisher({ sdp: offer.sdp || '', tracks: trackInfos, @@ -579,58 +567,11 @@ export class Publisher { const transceiver = this.transceiverCache.get(TrackType.SCREEN_SHARE_AUDIO); if (!transceiver) return sdp; - const mid = this.extractMid(transceiver, sdp, TrackType.SCREEN_SHARE_AUDIO); - return enableHighQualityAudio(sdp, mid); - }; - - private mungeCodecs = (sdp?: string) => { - if (sdp) { - sdp = toggleDtx(sdp, this.isDtxEnabled); - } - return sdp; - }; - - private extractMid = ( - transceiver: RTCRtpTransceiver, - sdp: string | undefined, - trackType: TrackType, - ): string => { - if (transceiver.mid) return transceiver.mid; - - if (!sdp) { - this.logger('warn', 'No SDP found. Returning empty mid'); - return ''; - } - - this.logger( - 'debug', - `No 'mid' found for track. Trying to find it from the Offer SDP`, + const transceiverInitIndex = this.transceiverInitOrder.indexOf( + TrackType.SCREEN_SHARE_AUDIO, ); - - const track = transceiver.sender.track!; - const parsedSdp = SDP.parse(sdp); - const media = parsedSdp.media.find((m) => { - return ( - m.type === track.kind && - // if `msid` is not present, we assume that the track is the first one - (m.msid?.includes(track.id) ?? true) - ); - }); - if (typeof media?.mid === 'undefined') { - this.logger( - 'debug', - `No mid found in SDP for track type ${track.kind} and id ${track.id}. Attempting to find it heuristically`, - ); - - const heuristicMid = this.transceiverInitOrder.indexOf(trackType); - if (heuristicMid !== -1) { - return String(heuristicMid); - } - - this.logger('debug', 'No heuristic mid found. Returning empty mid'); - return ''; - } - return String(media.mid); + const mid = extractMid(transceiver, transceiverInitIndex, sdp); + return enableHighQualityAudio(sdp, mid); }; /** @@ -669,7 +610,7 @@ export class Publisher { rid: optimalLayer.rid || '', bitrate: optimalLayer.maxBitrate || 0, fps: optimalLayer.maxFramerate || 0, - quality: this.ridToVideoQuality(optimalLayer.rid || ''), + quality: ridToVideoQuality(optimalLayer.rid || ''), videoDimension: { width: optimalLayer.width, height: optimalLayer.height, @@ -683,13 +624,13 @@ export class Publisher { const trackSettings = track.getSettings(); const isStereo = isAudioTrack && trackSettings.channelCount === 2; - + const transceiverInitIndex = + this.transceiverInitOrder.indexOf(trackType); return { trackId: track.id, layers: layers, trackType, - mid: this.extractMid(transceiver, sdp, trackType), - + mid: extractMid(transceiver, transceiverInitIndex, sdp), stereo: isStereo, dtx: isAudioTrack && this.isDtxEnabled, red: isAudioTrack && this.isRedEnabled, @@ -750,12 +691,4 @@ export class Publisher { private onSignalingStateChange = () => { this.logger('debug', `Signaling state changed`, this.pc.signalingState); }; - - private ridToVideoQuality = (rid: string): VideoQuality => { - return rid === 'q' - ? VideoQuality.LOW_UNSPECIFIED - : rid === 'h' - ? VideoQuality.MID - : VideoQuality.HIGH; // default to HIGH - }; } diff --git a/packages/client/src/rtc/__tests__/videoLayers.test.ts b/packages/client/src/rtc/__tests__/videoLayers.test.ts index 67a84cb7fb..49a0f80339 100644 --- a/packages/client/src/rtc/__tests__/videoLayers.test.ts +++ b/packages/client/src/rtc/__tests__/videoLayers.test.ts @@ -4,7 +4,11 @@ import { findOptimalScreenSharingLayers, findOptimalVideoLayers, getComputedMaxBitrate, + OptimalVideoLayer, + ridToVideoQuality, + toSvcEncodings, } from '../videoLayers'; +import { VideoQuality } from '../../gen/video/sfu/models/models'; describe('videoLayers', () => { it('should find optimal screen sharing layers', () => { @@ -152,16 +156,34 @@ describe('videoLayers', () => { expect(layers[2].rid).toBe('f'); }); + it('should map rids to VideoQuality', () => { + expect(ridToVideoQuality('q')).toBe(VideoQuality.LOW_UNSPECIFIED); + expect(ridToVideoQuality('h')).toBe(VideoQuality.MID); + expect(ridToVideoQuality('f')).toBe(VideoQuality.HIGH); + expect(ridToVideoQuality('')).toBe(VideoQuality.HIGH); + }); + + it('should map OptimalVideoLayer to SVC encodings', () => { + const layers: Array> = [ + { rid: 'f', width: 1920, height: 1080, maxBitrate: 3000000 }, + { rid: 'h', width: 960, height: 540, maxBitrate: 750000 }, + { rid: 'q', width: 480, height: 270, maxBitrate: 187500 }, + ]; + + const svcLayers = toSvcEncodings(layers as OptimalVideoLayer[]); + expect(svcLayers.length).toBe(1); + expect(svcLayers[0]).toEqual({ + rid: 'q', + width: 1920, + height: 1080, + maxBitrate: 3000000, + }); + }); + describe('getComputedMaxBitrate', () => { it('should scale target bitrate down if resolution is smaller than target resolution', () => { const targetResolution = { width: 1920, height: 1080, bitrate: 3000000 }; - const scaledBitrate = getComputedMaxBitrate( - targetResolution, - 1280, - 720, - undefined, - undefined, - ); + const scaledBitrate = getComputedMaxBitrate(targetResolution, 1280, 720); expect(scaledBitrate).toBe(1333333); }); @@ -171,13 +193,7 @@ describe('videoLayers', () => { const targetBitrates = ['f', 'h', 'q'].map((rid) => { const width = targetResolution.width / downscaleFactor; const height = targetResolution.height / downscaleFactor; - const bitrate = getComputedMaxBitrate( - targetResolution, - width, - height, - undefined, - undefined, - ); + const bitrate = getComputedMaxBitrate(targetResolution, width, height); downscaleFactor *= 2; return { rid, diff --git a/packages/client/src/rtc/videoLayers.ts b/packages/client/src/rtc/videoLayers.ts index c739d71b1b..e07f013979 100644 --- a/packages/client/src/rtc/videoLayers.ts +++ b/packages/client/src/rtc/videoLayers.ts @@ -2,6 +2,7 @@ import { PreferredCodec, PublishOptions } from '../types'; import { TargetResolutionResponse } from '../gen/shims'; import { isSvcCodec } from './codecs'; import { getOptimalBitrate } from './bitrateLookup'; +import { VideoQuality } from '../gen/video/sfu/models/models'; export type OptimalVideoLayer = RTCRtpEncodingParameters & { width: number; @@ -35,6 +36,17 @@ export const toSvcEncodings = (layers: OptimalVideoLayer[] | undefined) => { return layers?.filter((l) => l.rid === 'f').map((l) => ({ ...l, rid: 'q' })); }; +/** + * Converts the rid to a video quality. + */ +export const ridToVideoQuality = (rid: string): VideoQuality => { + return rid === 'q' + ? VideoQuality.LOW_UNSPECIFIED + : rid === 'h' + ? VideoQuality.MID + : VideoQuality.HIGH; // default to HIGH +}; + /** * Determines the most optimal video layers for simulcasting * for the given track. From 94ebaaa36e9d6952d6c80b2304a64967a4bf7248 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 12:56:11 +0200 Subject: [PATCH 27/33] fix: use vp8 as a default codec for Android --- packages/client/src/rtc/codecs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/rtc/codecs.ts b/packages/client/src/rtc/codecs.ts index 7a7899189b..8c498c029c 100644 --- a/packages/client/src/rtc/codecs.ts +++ b/packages/client/src/rtc/codecs.ts @@ -102,7 +102,7 @@ export const getOptimalVideoCodec = ( ): PreferredCodec => { if (isReactNative()) { const os = getOSInfo()?.name.toLowerCase(); - if (os === 'android') return preferredOr(preferredCodec, 'vp9'); + if (os === 'android') return preferredOr(preferredCodec, 'vp8'); if (os === 'ios' || os === 'ipados') return 'h264'; return preferredOr(preferredCodec, 'h264'); } From a516f474e51bcfd2a317ea27c93f5292094b8779 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 13:10:48 +0200 Subject: [PATCH 28/33] chore: update roadmaps --- packages/react-native-sdk/README.md | 8 ++++---- packages/react-sdk/README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/react-native-sdk/README.md b/packages/react-native-sdk/README.md index 032768f1d5..d0b264ee6a 100644 --- a/packages/react-native-sdk/README.md +++ b/packages/react-native-sdk/README.md @@ -136,10 +136,10 @@ Stream's video roadmap and changelog are available [here](https://github.com/Get - [x] Speaking while muted - [x] Demo app on play-store and app-store - [x] Transcriptions -- [ ] Speaker management (needs docs) -- [ ] PiP on iOS -- [ ] Audio & Video filters -- [ ] CPU usage improvement +- [x] Speaker management +- [x] PiP on iOS +- [x] Audio & Video filters +- [x] CPU usage improvement - [ ] Analytics Integration - [ ] Long press to focus - [ ] Dynascale 2.0 diff --git a/packages/react-sdk/README.md b/packages/react-sdk/README.md index 71a1d2e287..bbcce1e1e1 100644 --- a/packages/react-sdk/README.md +++ b/packages/react-sdk/README.md @@ -89,7 +89,7 @@ Here are some of the features we support: - [ ] Closed captions - [ ] Query call session endpoint - [ ] Logging 2.0 -- [ ] Hardware-accelerated video encoding on supported platforms +- [x] Hardware-accelerated video encoding on supported platforms - [ ] Dynascale 2.0 (codec switching) - [ ] E2E testing platform - [ ] Dynascale: turn off incoming video when the browser is in the background From a1a94183d5d98837137a40dc825ff877fb49d651 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 13:13:47 +0200 Subject: [PATCH 29/33] chore: missing checksums --- yarn.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yarn.lock b/yarn.lock index 3a257595ea..2de968f2d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -26190,6 +26190,7 @@ __metadata: bin: tsc: bin/tsc tsserver: bin/tsserver + checksum: ab417a2f398380c90a6cf5a5f74badd17866adf57f1165617d6a551f059c3ba0a3e4da0d147b3ac5681db9ac76a303c5876394b13b3de75fdd5b1eaa06181c9d languageName: node linkType: hard @@ -26199,6 +26200,7 @@ __metadata: bin: tsc: bin/tsc tsserver: bin/tsserver + checksum: 9d89bac0de650e15d6846485f238d1e65f1013f2c260d9e53e86a1da6ecf8109d9fad9402575c5c36a6592dc5d4370db090e12971c8630ae84453654baabb6b4 languageName: node linkType: hard From 5a15195abd41f708da1e4e0c8b9dffd07c2f15ef Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 13:23:20 +0200 Subject: [PATCH 30/33] chore: backport the fixes of #1437 --- packages/client/src/devices/CameraManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/client/src/devices/CameraManager.ts b/packages/client/src/devices/CameraManager.ts index 847bba9437..33b1a55553 100644 --- a/packages/client/src/devices/CameraManager.ts +++ b/packages/client/src/devices/CameraManager.ts @@ -58,10 +58,10 @@ export class CameraManager extends InputMediaDeviceManager { this.logger('warn', 'could not apply target resolution', error); } } - if (this.enabled) { - const { width, height } = this.state - .mediaStream!.getVideoTracks()[0] - ?.getSettings(); + if (this.enabled && this.state.mediaStream) { + const [videoTrack] = this.state.mediaStream.getVideoTracks(); + if (!videoTrack) return; + const { width, height } = videoTrack.getSettings(); if ( width !== this.targetResolution.width || height !== this.targetResolution.height From 41a9a5bf81baceab839c22396575e5b4a5fcadfb Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 13:26:25 +0200 Subject: [PATCH 31/33] chore: update roadmap --- packages/react-native-sdk/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/react-native-sdk/README.md b/packages/react-native-sdk/README.md index d0b264ee6a..7185d0136a 100644 --- a/packages/react-native-sdk/README.md +++ b/packages/react-native-sdk/README.md @@ -108,9 +108,9 @@ Stream's video roadmap and changelog are available [here](https://github.com/Get - [x] Video Rooms Tutorial - [x] Audio Rooms Tutorial - [x] Livestream Tutorial - - [ ] Core + - [x] Core - [x] Camera & Microphone - - [ ] Advanced + - [x] Advanced - [x] Chat Integration - [x] Internationalization - [x] Push Notification (validate) @@ -125,7 +125,7 @@ Stream's video roadmap and changelog are available [here](https://github.com/Get - [x] Expo Support -### 0.5 Milestones +### Milestones - [x] Regular Push Notification for Vanilla React Native - [x] Ringing and Regular Push Notification for Expo @@ -138,8 +138,9 @@ Stream's video roadmap and changelog are available [here](https://github.com/Get - [x] Transcriptions - [x] Speaker management - [x] PiP on iOS -- [x] Audio & Video filters +- [x] Video filters - [x] CPU usage improvement +- [ ] Audio filters and Noise Cancellation - [ ] Analytics Integration - [ ] Long press to focus - [ ] Dynascale 2.0 From 2c4486128b410f034ab9a97e0a5727080ccbcd89 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 14:11:03 +0200 Subject: [PATCH 32/33] chore: restore auto-release --- .github/workflows/version-and-release.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/version-and-release.yml b/.github/workflows/version-and-release.yml index 8040c9a49e..f6ea7248b4 100644 --- a/.github/workflows/version-and-release.yml +++ b/.github/workflows/version-and-release.yml @@ -4,11 +4,11 @@ env: STREAM_SECRET: ${{ secrets.CLIENT_TEST_SECRET }} on: - # push: - # branches: - # - main - # paths: - # - 'packages/**' + push: + branches: + - main + paths: + - 'packages/**' workflow_dispatch: From 6b84d4b673405f95b59f32519dcc4b1f297e4e78 Mon Sep 17 00:00:00 2001 From: Oliver Lazoroski Date: Wed, 16 Oct 2024 14:13:11 +0200 Subject: [PATCH 33/33] chore: restore auto-release --- .github/workflows/version-and-release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/version-and-release.yml b/.github/workflows/version-and-release.yml index f6ea7248b4..451da6e716 100644 --- a/.github/workflows/version-and-release.yml +++ b/.github/workflows/version-and-release.yml @@ -5,10 +5,10 @@ env: on: push: - branches: - - main - paths: - - 'packages/**' + branches: + - main + paths: + - 'packages/**' workflow_dispatch: