feat(svc-codec): VP9 and AV1 support

.github/workflows/version-and-release.yml

@@ -4,11 +4,11 @@ env:
   STREAM_SECRET: ${{ secrets.CLIENT_TEST_SECRET }}
 
 on:
-  # push:
-  #  branches:
-  #    - main
-  #  paths:
-  #    - 'packages/**'
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/**'
 
   workflow_dispatch:
 

packages/client/src/Call.ts

@@ -87,10 +87,7 @@ import {
   VideoTrackType,
 } from './types';
 import { BehaviorSubject, Subject, takeWhile } from 'rxjs';
-import {
-  ReconnectDetails,
-  VideoLayerSetting,
-} from './gen/video/sfu/event/events';
+import { ReconnectDetails } from './gen/video/sfu/event/events';
 import {
   ClientDetails,
   TrackType,
@@ -202,6 +199,7 @@ export class Call {
    */
   private readonly dispatcher = new Dispatcher();
 
+  private publishOptions?: PublishOptions;
   private statsReporter?: StatsReporter;
   private sfuStatsReporter?: SfuStatsReporter;
   private dropTimeout: ReturnType<typeof setTimeout> | undefined;
@@ -1292,19 +1290,12 @@ export class Call {
           break;
         case TrackType.VIDEO:
           const videoStream = this.camera.state.mediaStream;
-          if (videoStream) {
-            await this.publishVideoStream(
-              videoStream,
-              this.camera.publishOptions,
-            );
-          }
+          if (videoStream) await this.publishVideoStream(videoStream);
           break;
         case TrackType.SCREEN_SHARE:
           const screenShareStream = this.screenShare.state.mediaStream;
           if (screenShareStream) {
-            await this.publishScreenShareStream(screenShareStream, {
-              screenShareSettings: this.screenShare.getSettings(),
-            });
+            await this.publishScreenShareStream(screenShareStream);
           }
           break;
         // screen share audio can't exist without a screen share, so we handle it there
@@ -1336,12 +1327,8 @@ export class Call {
    * The previous video stream will be stopped.
    *
    * @param videoStream the video stream to publish.
-   * @param opts the options to use when publishing the stream.
    */
-  publishVideoStream = async (
-    videoStream: MediaStream,
-    opts: PublishOptions = {},
-  ) => {
+  publishVideoStream = async (videoStream: MediaStream) => {
     if (!this.sfuClient) throw new Error(`Call not joined yet.`);
     // joining is in progress, and we should wait until the client is ready
     await this.sfuClient.joinTask;
@@ -1363,7 +1350,7 @@ export class Call {
       videoStream,
       videoTrack,
       TrackType.VIDEO,
-      opts,
+      this.publishOptions,
     );
   };
 
@@ -1407,12 +1394,8 @@ export class Call {
    * The previous screen-share stream will be stopped.
    *
    * @param screenShareStream the screen-share stream to publish.
-   * @param opts the options to use when publishing the stream.
    */
-  publishScreenShareStream = async (
-    screenShareStream: MediaStream,
-    opts: PublishOptions = {},
-  ) => {
+  publishScreenShareStream = async (screenShareStream: MediaStream) => {
     if (!this.sfuClient) throw new Error(`Call not joined yet.`);
     // joining is in progress, and we should wait until the client is ready
     await this.sfuClient.joinTask;
@@ -1431,6 +1414,9 @@ export class Call {
     if (!this.trackPublishOrder.includes(TrackType.SCREEN_SHARE)) {
       this.trackPublishOrder.push(TrackType.SCREEN_SHARE);
     }
+    const opts: PublishOptions = {
+      screenShareSettings: this.screenShare.getSettings(),
+    };
     await this.publisher.publishStream(
       screenShareStream,
       screenShareTrack,
@@ -1467,6 +1453,16 @@ export class Call {
     await this.publisher?.unpublishStream(trackType, stopTrack);
   };
 
+  /**
+   * Updates the preferred publishing options
+   *
+   * @internal
+   * @param options the options to use.
+   */
+  updatePublishOptions(options: PublishOptions) {
+    this.publishOptions = { ...this.publishOptions, ...options };
+  }
+
   /**
    * Notifies the SFU that a noise cancellation process has started.
    *
@@ -1529,16 +1525,6 @@ export class Call {
     return this.state.setSortParticipantsBy(criteria);
   };
 
-  /**
-   * Updates the list of video layers to publish.
-   *
-   * @internal
-   * @param enabledLayers the list of layers to enable.
-   */
-  updatePublishQuality = async (enabledLayers: VideoLayerSetting[]) => {
-    return this.publisher?.updateVideoPublishQuality(enabledLayers);
-  };
-
   /**
    * Sends a reaction to the other call participants.
    *
@@ -2098,10 +2084,7 @@ export class Call {
         this.camera.state.mediaStream &&
         !this.publisher?.isPublishing(TrackType.VIDEO)
       ) {
-        await this.publishVideoStream(
-          this.camera.state.mediaStream,
-          this.camera.publishOptions,
-        );
+        await this.publishVideoStream(this.camera.state.mediaStream);
       }
 
       // Start camera if backend config specifies, and there is no local setting

packages/client/src/devices/CameraManager.ts

@@ -4,7 +4,7 @@ import { CameraDirection, CameraManagerState } from './CameraManagerState';
 import { InputMediaDeviceManager } from './InputMediaDeviceManager';
 import { getVideoDevices, getVideoStream } from './devices';
 import { TrackType } from '../gen/video/sfu/models/models';
-import { PreferredCodec, PublishOptions } from '../types';
+import { PreferredCodec } from '../types';
 import { isMobile } from '../compatibility';
 
 export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
@@ -13,13 +13,6 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
     height: 720,
   };
 
-  /**
-   * The options to use when publishing the video stream.
-   *
-   * @internal
-   */
-  publishOptions: PublishOptions | undefined;
-
   /**
    * Constructs a new CameraManager.
    *
@@ -70,10 +63,10 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
         this.logger('warn', 'could not apply target resolution', error);
       }
     }
-    if (this.enabled) {
-      const { width, height } = this.state
-        .mediaStream!.getVideoTracks()[0]
-        ?.getSettings();
+    if (this.enabled && this.state.mediaStream) {
+      const [videoTrack] = this.state.mediaStream.getVideoTracks();
+      if (!videoTrack) return;
+      const { width, height } = videoTrack.getSettings();
       if (
         width !== this.targetResolution.width ||
         height !== this.targetResolution.height
@@ -91,38 +84,11 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
    * Sets the preferred codec for encoding the video.
    *
    * @internal internal use only, not part of the public API.
+   * @deprecated use {@link call.updatePublishOptions} instead.
    * @param codec the codec to use for encoding the video.
    */
   setPreferredCodec(codec: PreferredCodec | undefined) {
-    this.updatePublishOptions({ preferredCodec: codec });
-  }
-
-  /**
-   * Updates the preferred publish options for the video stream.
-   *
-   * @internal
-   * @param options the options to use.
-   */
-  updatePublishOptions(options: PublishOptions) {
-    this.publishOptions = { ...this.publishOptions, ...options };
-  }
-
-  /**
-   * Returns the capture resolution of the camera.
-   */
-  getCaptureResolution() {
-    const { mediaStream } = this.state;
-    if (!mediaStream) return;
-
-    const [videoTrack] = mediaStream.getVideoTracks();
-    if (!videoTrack) return;
-
-    const settings = videoTrack.getSettings();
-    return {
-      width: settings.width,
-      height: settings.height,
-      frameRate: settings.frameRate,
-    };
+    this.call.updatePublishOptions({ preferredCodec: codec });
   }
 
   protected getDevices(): Observable<MediaDeviceInfo[]> {
@@ -144,7 +110,7 @@ export class CameraManager extends InputMediaDeviceManager<CameraManagerState> {
   }
 
   protected publishStream(stream: MediaStream): Promise<void> {
-    return this.call.publishVideoStream(stream, this.publishOptions);
+    return this.call.publishVideoStream(stream);
   }
 
   protected stopPublishStream(stopTracks: boolean): Promise<void> {

packages/client/src/devices/ScreenShareManager.ts

@@ -80,9 +80,7 @@ export class ScreenShareManager extends InputMediaDeviceManager<
   }
 
   protected publishStream(stream: MediaStream): Promise<void> {
-    return this.call.publishScreenShareStream(stream, {
-      screenShareSettings: this.state.settings,
-    });
+    return this.call.publishScreenShareStream(stream);
   }
 
   protected async stopPublishStream(stopTracks: boolean): Promise<void> {

packages/client/src/devices/__tests__/CameraManager.test.ts

@@ -89,21 +89,6 @@ describe('CameraManager', () => {
 
     expect(manager['call'].publishVideoStream).toHaveBeenCalledWith(
       manager.state.mediaStream,
-      undefined,
-    );
-  });
-
-  it('publish stream with preferred codec', async () => {
-    manager['call'].state.setCallingState(CallingState.JOINED);
-    manager.setPreferredCodec('h264');
-
-    await manager.enable();
-
-    expect(manager['call'].publishVideoStream).toHaveBeenCalledWith(
-      manager.state.mediaStream,
-      {
-        preferredCodec: 'h264',
-      },
     );
   });
 

packages/client/src/devices/__tests__/ScreenShareManager.test.ts

@@ -115,20 +115,6 @@ describe('ScreenShareManager', () => {
     await manager.enable();
     expect(call.publishScreenShareStream).toHaveBeenCalledWith(
       manager.state.mediaStream,
-      { screenShareSettings: undefined },
-    );
-  });
-
-  it('publishes screen share stream with settings', async () => {
-    const call = manager['call'];
-    call.state.setCallingState(CallingState.JOINED);
-
-    manager.setSettings({ maxFramerate: 15, maxBitrate: 1000 });
-
-    await manager.enable();
-    expect(call.publishScreenShareStream).toHaveBeenCalledWith(
-      manager.state.mediaStream,
-      { screenShareSettings: { maxFramerate: 15, maxBitrate: 1000 } },
     );
   });
 

packages/client/src/events/callEventHandlers.ts

@@ -7,7 +7,6 @@ import {
   watchCallEnded,
   watchCallGrantsUpdated,
   watchCallRejected,
-  watchChangePublishQuality,
   watchConnectionQualityChanged,
   watchDominantSpeakerChanged,
   watchLiveEnded,
@@ -46,7 +45,6 @@ export const registerEventHandlers = (call: Call, dispatcher: Dispatcher) => {
 
     watchLiveEnded(dispatcher, call),
     watchSfuErrorReports(dispatcher),
-    watchChangePublishQuality(dispatcher, call),
     watchConnectionQualityChanged(dispatcher, state),
     watchParticipantCountChanged(dispatcher, state),
 

packages/client/src/events/internal.ts

@@ -10,22 +10,6 @@ import {
 } from '../gen/video/sfu/models/models';
 import { OwnCapability } from '../gen/coordinator';
 
-/**
- * An event responder which handles the `changePublishQuality` event.
- */
-export const watchChangePublishQuality = (
-  dispatcher: Dispatcher,
-  call: Call,
-) => {
-  return dispatcher.on('changePublishQuality', (e) => {
-    const { videoSenders } = e;
-    videoSenders.forEach((videoSender) => {
-      const { layers } = videoSender;
-      call.updatePublishQuality(layers.filter((l) => l.active));
-    });
-  });
-};
-
 export const watchConnectionQualityChanged = (
   dispatcher: Dispatcher,
   state: CallState,