Allow configurable asset paths for task vision assets (#35)

.changeset/honest-kings-raise.md

@@ -0,0 +1,5 @@
+---
+"@livekit/track-processors": patch
+---
+
+Allow configurable asset paths for task vision assets

example/sample.ts

@@ -180,7 +180,7 @@ const appActions = {
     window.currentRoom = room;
     setButtonsForState(true);
 
-    room.participants.forEach((participant) => {
+    room.remoteParticipants.forEach((participant) => {
       participantConnected(participant);
     });
     participantConnected(room.localParticipant);
@@ -221,7 +221,7 @@ const appActions = {
   },
 
   flipVideo: () => {
-    const videoPub = currentRoom?.localParticipant.getTrack(Track.Source.Camera);
+    const videoPub = currentRoom?.localParticipant.getTrackPublication(Track.Source.Camera);
     if (!videoPub) {
       return;
     }
@@ -244,7 +244,7 @@ const appActions = {
     setButtonDisabled('toggle-blur-button', true);
 
     try {
-      const camTrack = currentRoom.localParticipant.getTrack(Track.Source.Camera)!
+      const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!
         .track as LocalVideoTrack;
       if (camTrack.getProcessor()?.name !== 'background-blur') {
         await camTrack.setProcessor(state.blur);
@@ -264,7 +264,7 @@ const appActions = {
     if (!currentRoom) return;
     setButtonDisabled('toggle-blur-button', true);
     try {
-      const camTrack = currentRoom.localParticipant.getTrack(Track.Source.Camera)!
+      const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!
         .track as LocalVideoTrack;
       if (camTrack.getProcessor()?.name !== 'virtual-background') {
         await camTrack.setProcessor(state.virtualBackground);
@@ -284,7 +284,7 @@ const appActions = {
     if (!currentRoom) return;
 
     try {
-      const camTrack = currentRoom.localParticipant.getTrack(Track.Source.Camera)!
+      const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)!
         .track as LocalVideoTrack;
       await state.virtualBackground.updateTransformerOptions({ imagePath });
       if (camTrack.getProcessor()?.name !== 'virtual-background') {
@@ -369,7 +369,7 @@ function handleRoomDisconnect(reason?: DisconnectReason) {
   appendLog('disconnected from room', { reason });
   setButtonsForState(false);
   renderParticipant(currentRoom.localParticipant, true);
-  currentRoom.participants.forEach((p) => {
+  currentRoom.remoteParticipants.forEach((p) => {
     renderParticipant(p, true);
   });
 
@@ -469,8 +469,8 @@ function renderParticipant(participant: Participant, remove: boolean = false) {
   }
   const micElm = $(`mic-${identity}`)!;
   const signalElm = $(`signal-${identity}`)!;
-  const cameraPub = participant.getTrack(Track.Source.Camera);
-  const micPub = participant.getTrack(Track.Source.Microphone);
+  const cameraPub = participant.getTrackPublication(Track.Source.Camera);
+  const micPub = participant.getTrackPublication(Track.Source.Microphone);
   if (participant.isSpeaking) {
     div!.classList.add('speaking');
   } else {
@@ -553,13 +553,13 @@ function renderBitrate() {
   if (!currentRoom || currentRoom.state !== ConnectionState.Connected) {
     return;
   }
-  const participants: Participant[] = [...currentRoom.participants.values()];
+  const participants: Participant[] = [...currentRoom.remoteParticipants.values()];
   participants.push(currentRoom.localParticipant);
 
   for (const p of participants) {
     const elm = $(`bitrate-${p.identity}`);
     let totalBitrate = 0;
-    for (const t of p.tracks.values()) {
+    for (const t of p.trackPublications.values()) {
       if (t.track) {
         totalBitrate += t.track.currentBitrate;
       }

package.json

@@ -40,7 +40,6 @@
     "eslint-config-prettier": "8.8.0",
     "eslint-plugin-ecmascript-compat": "^3.0.0",
     "eslint-plugin-import": "2.27.5",
-    "livekit-client": "^1.12.0",
     "prettier": "^2.8.8",
     "tsup": "^7.1.0",
     "typescript": "^5.0.4",

src/transformers/BackgroundTransformer.ts

@@ -8,7 +8,10 @@ export type SegmenterOptions = Partial<vision.ImageSegmenterOptions['baseOptions
 export type BackgroundOptions = {
   blurRadius?: number;
   imagePath?: string;
+  /** cannot be updated through the `update` method, needs a restart */
   segmenterOptions?: SegmenterOptions;
+  /** cannot be updated through the `update` method, needs a restart */
+  assetPaths?: { tasksVisionFileSet?: string; modelAssetPath?: string };
 };
 
 export default class BackgroundProcessor extends VideoTransformer<BackgroundOptions> {
@@ -36,12 +39,14 @@ export default class BackgroundProcessor extends VideoTransformer<BackgroundOpti
     await super.init({ outputCanvas, inputElement: inputVideo });
 
     const fileSet = await vision.FilesetResolver.forVisionTasks(
-      `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,
+      this.options.assetPaths?.tasksVisionFileSet ??
+        `https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@${dependencies['@mediapipe/tasks-vision']}/wasm`,
     );
 
     this.imageSegmenter = await vision.ImageSegmenter.createFromOptions(fileSet, {
       baseOptions: {
         modelAssetPath:
+          this.options.assetPaths?.modelAssetPath ??
           'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/latest/selfie_segmenter.tflite',
         delegate: 'GPU',
         ...this.options.segmenterOptions,