Stronger kind type for Tracks to improve processor support (#1033)

* Stronger kind type for Tracks to improve processor support * Create honest-fireants-perform.md * add onPublish handler * simplify
livekit · Feb 19, 2024 · 8b031b0 · 8b031b0
1 parent 7086889
commit 8b031b0
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 45 deletions.
diff --git a/.changeset/honest-fireants-perform.md b/.changeset/honest-fireants-perform.md
@@ -0,0 +1,5 @@
+---
+"livekit-client": patch
+---
+
+Stronger kind type for Tracks to improve processor support
diff --git a/src/room/Room.ts b/src/room/Room.ts
@@ -1776,7 +1776,10 @@ class Room extends (EventEmitter as new () => TypedEmitter<RoomEventCallbacks>)
   };
 
   private onLocalTrackPublished = async (pub: LocalTrackPublication) => {
+    pub.track?.getProcessor()?.onPublish?.(this);
+
     this.emit(RoomEvent.LocalTrackPublished, pub, this.localParticipant);
+
     if (pub.track instanceof LocalAudioTrack) {
       const trackIsSilent = await pub.track.checkForSilence();
       if (trackIsSilent) {

diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts
@@ -888,7 +888,6 @@ export default class LocalParticipant extends Participant {
     }
 
     this.addTrackPublication(publication);
-
     // send event for publication
     this.emit(ParticipantEvent.LocalTrackPublished, publication);
     return publication;

diff --git a/src/room/track/LocalAudioTrack.ts b/src/room/track/LocalAudioTrack.ts
@@ -6,17 +6,17 @@ import { isWeb, unwrapConstraint } from '../utils';
 import LocalTrack from './LocalTrack';
 import { Track } from './Track';
 import type { AudioCaptureOptions } from './options';
-import type { TrackProcessor } from './processor/types';
+import type { AudioProcessorOptions, TrackProcessor } from './processor/types';
 import { constraintsForOptions, detectSilence } from './utils';
 
-export default class LocalAudioTrack extends LocalTrack {
+export default class LocalAudioTrack extends LocalTrack<Track.Kind.Audio> {
   /** @internal */
   stopOnMute: boolean = false;
 
-  private audioContext?: AudioContext;
-
   private prevStats?: AudioSenderStats;
 
+  protected processor?: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions> | undefined;
+
   /**
    *
    * @param mediaTrack
@@ -48,7 +48,7 @@ export default class LocalAudioTrack extends LocalTrack {
     );
   }
 
-  async mute(): Promise<LocalAudioTrack> {
+  async mute(): Promise<typeof this> {
     const unlock = await this.muteLock.lock();
     try {
       // disabled special handling as it will cause BT headsets to switch communication modes
@@ -64,7 +64,7 @@ export default class LocalAudioTrack extends LocalTrack {
     }
   }
 
-  async unmute(): Promise<LocalAudioTrack> {
+  async unmute(): Promise<typeof this> {
     const unlock = await this.muteLock.lock();
     try {
       const deviceHasChanged =
@@ -99,7 +99,7 @@ export default class LocalAudioTrack extends LocalTrack {
     await this.restart(constraints);
   }
 
-  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
+  protected async restart(constraints?: MediaTrackConstraints): Promise<typeof this> {
     const track = await super.restart(constraints);
     this.checkForSilence();
     return track;
@@ -139,7 +139,7 @@ export default class LocalAudioTrack extends LocalTrack {
     this.prevStats = stats;
   };
 
-  async setProcessor(processor: TrackProcessor<this['kind']>) {
+  async setProcessor(processor: TrackProcessor<Track.Kind.Audio, AudioProcessorOptions>) {
     const unlock = await this.processorLock.lock();
     try {
       if (!this.audioContext) {
@@ -150,9 +150,6 @@ export default class LocalAudioTrack extends LocalTrack {
       if (this.processor) {
         await this.stopProcessor();
       }
-      if (this.kind === 'unknown') {
-        throw TypeError('cannot set processor on track of unknown kind');
-      }
 
       const processorOptions = {
         kind: this.kind,

diff --git a/src/room/track/LocalTrack.ts b/src/room/track/LocalTrack.ts
@@ -11,7 +11,9 @@ import type { TrackProcessor } from './processor/types';
 
 const defaultDimensionsTimeout = 1000;
 
-export default abstract class LocalTrack extends Track {
+export default abstract class LocalTrack<
+  TrackKind extends Track.Kind = Track.Kind,
+> extends Track<TrackKind> {
   /** @internal */
   sender?: RTCRtpSender;
 
@@ -34,10 +36,12 @@ export default abstract class LocalTrack extends Track {
 
   protected processorElement?: HTMLMediaElement;
 
-  protected processor?: TrackProcessor<this['kind']>;
+  protected processor?: TrackProcessor<TrackKind, any>;
 
   protected processorLock: Mutex;
 
+  protected audioContext?: AudioContext;
+
   /**
    *
    * @param mediaTrack
@@ -47,7 +51,7 @@ export default abstract class LocalTrack extends Track {
    */
   protected constructor(
     mediaTrack: MediaStreamTrack,
-    kind: Track.Kind,
+    kind: TrackKind,
     constraints?: MediaTrackConstraints,
     userProvidedTrack = false,
     loggerOptions?: LoggerOptions,
@@ -128,21 +132,28 @@ export default abstract class LocalTrack extends Track {
       this._constraints = newTrack.getConstraints();
     }
     let processedTrack: MediaStreamTrack | undefined;
-    if (this.processor && newTrack && this.processorElement) {
-      this.log.debug('restarting processor', this.logContext);
-      if (this.kind === 'unknown') {
-        throw TypeError('cannot set processor on track of unknown kind');
-      }
+    if (this.processor && newTrack) {
+      const unlock = await this.processorLock.lock();
+      try {
+        this.log.debug('restarting processor', this.logContext);
+        if (this.kind === 'unknown') {
+          throw TypeError('cannot set processor on track of unknown kind');
+        }
 
-      attachToElement(newTrack, this.processorElement);
-      // ensure the processorElement itself stays muted
-      this.processorElement.muted = true;
-      await this.processor.restart({
-        track: newTrack,
-        kind: this.kind,
-        element: this.processorElement,
-      });
-      processedTrack = this.processor.processedTrack;
+        if (this.processorElement) {
+          attachToElement(newTrack, this.processorElement);
+          // ensure the processorElement itself stays muted
+          this.processorElement.muted = true;
+        }
+        await this.processor.restart({
+          track: newTrack,
+          kind: this.kind,
+          element: this.processorElement,
+        });
+        processedTrack = this.processor.processedTrack;
+      } finally {
+        unlock();
+      }
     }
     if (this.sender) {
       await this.sender.replaceTrack(processedTrack ?? newTrack);
@@ -200,17 +211,17 @@ export default abstract class LocalTrack extends Track {
     return DeviceManager.getInstance().normalizeDeviceId(kind, deviceId, groupId);
   }
 
-  async mute(): Promise<LocalTrack> {
+  async mute() {
     this.setTrackMuted(true);
     return this;
   }
 
-  async unmute(): Promise<LocalTrack> {
+  async unmute() {
     this.setTrackMuted(false);
     return this;
   }
 
-  async replaceTrack(track: MediaStreamTrack, userProvidedTrack = true): Promise<LocalTrack> {
+  async replaceTrack(track: MediaStreamTrack, userProvidedTrack = true) {
     if (!this.sender) {
       throw new TrackInvalidError('unable to replace an unpublished track');
     }
@@ -227,7 +238,7 @@ export default abstract class LocalTrack extends Track {
     return this;
   }
 
-  protected async restart(constraints?: MediaTrackConstraints): Promise<LocalTrack> {
+  protected async restart(constraints?: MediaTrackConstraints) {
     if (!constraints) {
       constraints = this._constraints;
     }
@@ -408,7 +419,7 @@ export default abstract class LocalTrack extends Track {
    * @param showProcessedStreamLocally
    * @returns
    */
-  async setProcessor(processor: TrackProcessor<this['kind']>, showProcessedStreamLocally = true) {
+  async setProcessor(processor: TrackProcessor<TrackKind>, showProcessedStreamLocally = true) {
     const unlock = await this.processorLock.lock();
     try {
       this.log.debug('setting up processor', this.logContext);
@@ -418,7 +429,8 @@ export default abstract class LocalTrack extends Track {
       if (this.kind === 'unknown') {
         throw TypeError('cannot set processor on track of unknown kind');
       }
-      this.processorElement = this.processorElement ?? document.createElement(this.kind);
+      this.processorElement =
+        this.processorElement ?? (document.createElement(this.kind) as HTMLMediaElement);
 
       attachToElement(this._mediaStreamTrack, this.processorElement);
       this.processorElement.muted = true;
@@ -433,6 +445,7 @@ export default abstract class LocalTrack extends Track {
         kind: this.kind,
         track: this._mediaStreamTrack,
         element: this.processorElement,
+        audioContext: this.audioContext,
       };
 
       await processor.init(processorOptions);

diff --git a/src/room/track/LocalVideoTrack.ts b/src/room/track/LocalVideoTrack.ts
@@ -30,7 +30,7 @@ export class SimulcastTrackInfo {
 
 const refreshSubscribedCodecAfterNewCodec = 5000;
 
-export default class LocalVideoTrack extends LocalTrack {
+export default class LocalVideoTrack extends LocalTrack<Track.Kind.Video> {
   /* @internal */
   signalClient?: SignalClient;
 
@@ -115,7 +115,7 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
-  async mute(): Promise<LocalVideoTrack> {
+  async mute(): Promise<typeof this> {
     const unlock = await this.muteLock.lock();
     try {
       if (this.source === Track.Source.Camera && !this.isUserProvided) {
@@ -130,7 +130,7 @@ export default class LocalVideoTrack extends LocalTrack {
     }
   }
 
-  async unmute(): Promise<LocalVideoTrack> {
+  async unmute(): Promise<typeof this> {
     const unlock = await this.muteLock.lock();
     try {
       if (this.source === Track.Source.Camera && !this.isUserProvided) {

diff --git a/src/room/track/RemoteAudioTrack.ts b/src/room/track/RemoteAudioTrack.ts
@@ -7,7 +7,7 @@ import RemoteTrack from './RemoteTrack';
 import { Track } from './Track';
 import type { AudioOutputOptions } from './options';
 
-export default class RemoteAudioTrack extends RemoteTrack {
+export default class RemoteAudioTrack extends RemoteTrack<Track.Kind.Audio> {
   private prevStats?: AudioReceiverStats;
 
   private elementVolume: number | undefined;

diff --git a/src/room/track/RemoteTrack.ts b/src/room/track/RemoteTrack.ts
@@ -3,14 +3,16 @@ import { monitorFrequency } from '../stats';
 import type { LoggerOptions } from '../types';
 import { Track } from './Track';
 
-export default abstract class RemoteTrack extends Track {
+export default abstract class RemoteTrack<
+  TrackKind extends Track.Kind = Track.Kind,
+> extends Track<TrackKind> {
   /** @internal */
   receiver?: RTCRtpReceiver;
 
   constructor(
     mediaTrack: MediaStreamTrack,
     sid: string,
-    kind: Track.Kind,
+    kind: TrackKind,
     receiver?: RTCRtpReceiver,
     loggerOptions?: LoggerOptions,
   ) {

diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts
@@ -12,7 +12,7 @@ import type { AdaptiveStreamSettings } from './types';
 
 const REACTION_DELAY = 100;
 
-export default class RemoteVideoTrack extends RemoteTrack {
+export default class RemoteVideoTrack extends RemoteTrack<Track.Kind.Video> {
   private prevStats?: VideoReceiverStats;
 
   private elementInfos: ElementInfo[] = [];

diff --git a/src/room/track/Track.ts b/src/room/track/Track.ts
@@ -24,8 +24,10 @@ export enum VideoQuality {
   MEDIUM = ProtoQuality.MEDIUM,
   HIGH = ProtoQuality.HIGH,
 }
-export abstract class Track extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
-  kind: Track.Kind;
+export abstract class Track<
+  TrackKind extends Track.Kind = Track.Kind,
+> extends (EventEmitter as new () => TypedEventEmitter<TrackEventCallbacks>) {
+  readonly kind: TrackKind;
 
   attachedElements: HTMLMediaElement[] = [];
 
@@ -67,7 +69,7 @@ export abstract class Track extends (EventEmitter as new () => TypedEventEmitter
 
   protected constructor(
     mediaTrack: MediaStreamTrack,
-    kind: Track.Kind,
+    kind: TrackKind,
     loggerOptions: LoggerOptions = {},
   ) {
     super();

diff --git a/src/room/track/processor/types.ts b/src/room/track/processor/types.ts
@@ -1,3 +1,4 @@
+import type Room from '../../Room';
 import type { Track } from '../Track';
 
 /**
@@ -7,6 +8,7 @@ export type ProcessorOptions<T extends Track.Kind> = {
   kind: T;
   track: MediaStreamTrack;
   element?: HTMLMediaElement;
+  audioContext?: AudioContext;
 };
 
 /**
@@ -33,4 +35,6 @@ export interface TrackProcessor<
   restart: (opts: U) => Promise<void>;
   destroy: () => Promise<void>;
   processedTrack?: MediaStreamTrack;
+  onPublish?: (room: Room) => Promise<void>;
+  onUnpublish?: () => Promise<void>;
 }