Allow processors to be set as part of track publish options (#1143)

* Allow processors to be set as part of track publish options * Create eight-pugs-cough.md * remove console log * specify options
livekit · May 31, 2024 · cf781d8 · cf781d8
1 parent a28b2c5
commit cf781d8
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 53 deletions.
diff --git a/.changeset/eight-pugs-cough.md b/.changeset/eight-pugs-cough.md
@@ -0,0 +1,5 @@
+---
+"livekit-client": minor
+---
+
+Allow processors to be set as part of track publish options
diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts
@@ -30,6 +30,7 @@ import type {
   VideoCaptureOptions,
 } from '../track/options';
 import { ScreenSharePresets, VideoPresets, isBackupCodec } from '../track/options';
+import type { TrackProcessor } from '../track/processor/types';
 import {
   constraintsForOptions,
   getLogContextFromTrack,
@@ -394,13 +395,13 @@ export default class LocalParticipant extends Participant {
    * @returns
    */
   async createTracks(options?: CreateLocalTracksOptions): Promise<LocalTrack[]> {
-    const opts = mergeDefaultOptions(
+    const mergedOptions = mergeDefaultOptions(
       options,
       this.roomOptions?.audioCaptureDefaults,
       this.roomOptions?.videoCaptureDefaults,
     );
 
-    const constraints = constraintsForOptions(opts);
+    const constraints = constraintsForOptions(mergedOptions);
     let stream: MediaStream | undefined;
     try {
       stream = await navigator.mediaDevices.getUserMedia(constraints);
@@ -425,29 +426,39 @@ export default class LocalParticipant extends Participant {
       this.cameraError = undefined;
     }
 
-    return stream.getTracks().map((mediaStreamTrack) => {
-      const isAudio = mediaStreamTrack.kind === 'audio';
-      let trackOptions = isAudio ? options!.audio : options!.video;
-      if (typeof trackOptions === 'boolean' || !trackOptions) {
-        trackOptions = {};
-      }
-      let trackConstraints: MediaTrackConstraints | undefined;
-      const conOrBool = isAudio ? constraints.audio : constraints.video;
-      if (typeof conOrBool !== 'boolean') {
-        trackConstraints = conOrBool;
-      }
-      const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints, {
-        loggerName: this.roomOptions.loggerName,
-        loggerContextCb: () => this.logContext,
-      });
-      if (track.kind === Track.Kind.Video) {
-        track.source = Track.Source.Camera;
-      } else if (track.kind === Track.Kind.Audio) {
-        track.source = Track.Source.Microphone;
-      }
-      track.mediaStream = stream;
-      return track;
-    });
+    return Promise.all(
+      stream.getTracks().map(async (mediaStreamTrack) => {
+        const isAudio = mediaStreamTrack.kind === 'audio';
+        let trackOptions = isAudio ? mergedOptions!.audio : mergedOptions!.video;
+        if (typeof trackOptions === 'boolean' || !trackOptions) {
+          trackOptions = {};
+        }
+        let trackConstraints: MediaTrackConstraints | undefined;
+        const conOrBool = isAudio ? constraints.audio : constraints.video;
+        if (typeof conOrBool !== 'boolean') {
+          trackConstraints = conOrBool;
+        }
+        const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints, {
+          loggerName: this.roomOptions.loggerName,
+          loggerContextCb: () => this.logContext,
+        });
+        if (track.kind === Track.Kind.Video) {
+          track.source = Track.Source.Camera;
+        } else if (track.kind === Track.Kind.Audio) {
+          track.source = Track.Source.Microphone;
+          track.setAudioContext(this.audioContext);
+        }
+        track.mediaStream = stream;
+        if (trackOptions.processor) {
+          if (track instanceof LocalAudioTrack) {
+            await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Audio>);
+          } else {
+            await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Video>);
+          }
+        }
+        return track;
+      }),
+    );
   }
 
   /**

diff --git a/src/room/track/LocalVideoTrack.ts b/src/room/track/LocalVideoTrack.ts
@@ -276,7 +276,10 @@ export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
     }
   }
 
-  async setProcessor(processor: TrackProcessor<Track.Kind>, showProcessedStreamLocally = true) {
+  async setProcessor(
+    processor: TrackProcessor<Track.Kind.Video>,
+    showProcessedStreamLocally = true,
+  ) {
     await super.setProcessor(processor, showProcessedStreamLocally);
 
     if (this.processor?.processedTrack) {

diff --git a/src/room/track/create.ts b/src/room/track/create.ts
@@ -14,6 +14,7 @@ import type {
   VideoCaptureOptions,
 } from './options';
 import { ScreenSharePresets } from './options';
+import type { TrackProcessor } from './processor/types';
 import {
   constraintsForOptions,
   mergeDefaultOptions,
@@ -51,35 +52,44 @@ export async function createLocalTracks(
   }
 
   const stream = await mediaPromise;
-  return stream.getTracks().map((mediaStreamTrack) => {
-    const isAudio = mediaStreamTrack.kind === 'audio';
-    let trackOptions = isAudio ? options!.audio : options!.video;
-    if (typeof trackOptions === 'boolean' || !trackOptions) {
-      trackOptions = {};
-    }
-    let trackConstraints: MediaTrackConstraints | undefined;
-    const conOrBool = isAudio ? constraints.audio : constraints.video;
-    if (typeof conOrBool !== 'boolean') {
-      trackConstraints = conOrBool;
-    }
+  return Promise.all(
+    stream.getTracks().map(async (mediaStreamTrack) => {
+      const isAudio = mediaStreamTrack.kind === 'audio';
+      let trackOptions = isAudio ? options!.audio : options!.video;
+      if (typeof trackOptions === 'boolean' || !trackOptions) {
+        trackOptions = {};
+      }
+      let trackConstraints: MediaTrackConstraints | undefined;
+      const conOrBool = isAudio ? constraints.audio : constraints.video;
+      if (typeof conOrBool !== 'boolean') {
+        trackConstraints = conOrBool;
+      }
 
-    // update the constraints with the device id the user gave permissions to in the permission prompt
-    // otherwise each track restart (e.g. mute - unmute) will try to initialize the device again -> causing additional permission prompts
-    if (trackConstraints) {
-      trackConstraints.deviceId = mediaStreamTrack.getSettings().deviceId;
-    } else {
-      trackConstraints = { deviceId: mediaStreamTrack.getSettings().deviceId };
-    }
+      // update the constraints with the device id the user gave permissions to in the permission prompt
+      // otherwise each track restart (e.g. mute - unmute) will try to initialize the device again -> causing additional permission prompts
+      if (trackConstraints) {
+        trackConstraints.deviceId = mediaStreamTrack.getSettings().deviceId;
+      } else {
+        trackConstraints = { deviceId: mediaStreamTrack.getSettings().deviceId };
+      }
 
-    const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
-    if (track.kind === Track.Kind.Video) {
-      track.source = Track.Source.Camera;
-    } else if (track.kind === Track.Kind.Audio) {
-      track.source = Track.Source.Microphone;
-    }
-    track.mediaStream = stream;
-    return track;
-  });
+      const track = mediaTrackToLocalTrack(mediaStreamTrack, trackConstraints);
+      if (track.kind === Track.Kind.Video) {
+        track.source = Track.Source.Camera;
+      } else if (track.kind === Track.Kind.Audio) {
+        track.source = Track.Source.Microphone;
+      }
+      track.mediaStream = stream;
+      if (trackOptions.processor) {
+        if (track instanceof LocalAudioTrack) {
+          await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Audio>);
+        } else if (track instanceof LocalVideoTrack) {
+          await track.setProcessor(trackOptions.processor as TrackProcessor<Track.Kind.Video>);
+        }
+      }
+      return track;
+    }),
+  );
 }
 
 /**

diff --git a/src/room/track/options.ts b/src/room/track/options.ts
@@ -1,4 +1,9 @@
 import type { Track } from './Track';
+import type {
+  AudioProcessorOptions,
+  TrackProcessor,
+  VideoProcessorOptions,
+} from './processor/types';
 
 export interface TrackPublishDefaults {
   /**
@@ -152,6 +157,11 @@ export interface VideoCaptureOptions {
   facingMode?: 'user' | 'environment' | 'left' | 'right';
 
   resolution?: VideoResolution;
+
+  /**
+   * initialize the track with a given processor
+   */
+  processor?: TrackProcessor<Track.Kind.Video, VideoProcessorOptions>;
 }
 
 export interface ScreenShareCaptureOptions {
@@ -245,6 +255,11 @@ export interface AudioCaptureOptions {
    * sample size or range of sample sizes which are acceptable and/or required.
    */
   sampleSize?: ConstrainULong;
+
+  /**
+   * initialize the track with a given processor
+   */
+  processor?: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>;
 }
 
 export interface AudioOutputOptions {