From 13ab415f2c5d8e38ac7207aae6b9c19c037b64ce Mon Sep 17 00:00:00 2001 From: lukasIO Date: Tue, 19 Mar 2024 19:48:30 +0100 Subject: [PATCH] Allow configurable asset paths for task vision assets (#35) --- .changeset/honest-kings-raise.md | 5 ++ example/sample.ts | 20 ++++---- package.json | 1 - pnpm-lock.yaml | 59 ++++++++++++----------- src/transformers/BackgroundTransformer.ts | 7 ++- 5 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 .changeset/honest-kings-raise.md diff --git a/.changeset/honest-kings-raise.md b/.changeset/honest-kings-raise.md new file mode 100644 index 0000000..475d1a9 --- /dev/null +++ b/.changeset/honest-kings-raise.md @@ -0,0 +1,5 @@ +--- +"@livekit/track-processors": patch +--- + +Allow configurable asset paths for task vision assets diff --git a/example/sample.ts b/example/sample.ts index f3359c4..20e690f 100644 --- a/example/sample.ts +++ b/example/sample.ts @@ -180,7 +180,7 @@ const appActions = { window.currentRoom = room; setButtonsForState(true); - room.participants.forEach((participant) => { + room.remoteParticipants.forEach((participant) => { participantConnected(participant); }); participantConnected(room.localParticipant); @@ -221,7 +221,7 @@ const appActions = { }, flipVideo: () => { - const videoPub = currentRoom?.localParticipant.getTrack(Track.Source.Camera); + const videoPub = currentRoom?.localParticipant.getTrackPublication(Track.Source.Camera); if (!videoPub) { return; } @@ -244,7 +244,7 @@ const appActions = { setButtonDisabled('toggle-blur-button', true); try { - const camTrack = currentRoom.localParticipant.getTrack(Track.Source.Camera)! + const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)! .track as LocalVideoTrack; if (camTrack.getProcessor()?.name !== 'background-blur') { await camTrack.setProcessor(state.blur); @@ -264,7 +264,7 @@ const appActions = { if (!currentRoom) return; setButtonDisabled('toggle-blur-button', true); try { - const camTrack = currentRoom.localParticipant.getTrack(Track.Source.Camera)! + const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)! .track as LocalVideoTrack; if (camTrack.getProcessor()?.name !== 'virtual-background') { await camTrack.setProcessor(state.virtualBackground); @@ -284,7 +284,7 @@ const appActions = { if (!currentRoom) return; try { - const camTrack = currentRoom.localParticipant.getTrack(Track.Source.Camera)! + const camTrack = currentRoom.localParticipant.getTrackPublication(Track.Source.Camera)! .track as LocalVideoTrack; await state.virtualBackground.updateTransformerOptions({ imagePath }); if (camTrack.getProcessor()?.name !== 'virtual-background') { @@ -369,7 +369,7 @@ function handleRoomDisconnect(reason?: DisconnectReason) { appendLog('disconnected from room', { reason }); setButtonsForState(false); renderParticipant(currentRoom.localParticipant, true); - currentRoom.participants.forEach((p) => { + currentRoom.remoteParticipants.forEach((p) => { renderParticipant(p, true); }); @@ -469,8 +469,8 @@ function renderParticipant(participant: Participant, remove: boolean = false) { } const micElm = $(`mic-${identity}`)!; const signalElm = $(`signal-${identity}`)!; - const cameraPub = participant.getTrack(Track.Source.Camera); - const micPub = participant.getTrack(Track.Source.Microphone); + const cameraPub = participant.getTrackPublication(Track.Source.Camera); + const micPub = participant.getTrackPublication(Track.Source.Microphone); if (participant.isSpeaking) { div!.classList.add('speaking'); } else { @@ -553,13 +553,13 @@ function renderBitrate() { if (!currentRoom || currentRoom.state !== ConnectionState.Connected) { return; } - const participants: Participant[] = [...currentRoom.participants.values()]; + const participants: Participant[] = [...currentRoom.remoteParticipants.values()]; participants.push(currentRoom.localParticipant); for (const p of participants) { const elm = $(`bitrate-${p.identity}`); let totalBitrate = 0; - for (const t of p.tracks.values()) { + for (const t of p.trackPublications.values()) { if (t.track) { totalBitrate += t.track.currentBitrate; } diff --git a/package.json b/package.json index 333520f..a7a65bf 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "eslint-config-prettier": "8.8.0", "eslint-plugin-ecmascript-compat": "^3.0.0", "eslint-plugin-import": "2.27.5", - "livekit-client": "^1.12.0", "prettier": "^2.8.8", "tsup": "^7.1.0", "typescript": "^5.0.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fc86ba4..226fa5e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ dependencies: '@mediapipe/tasks-vision': specifier: 0.10.9 version: 0.10.9 + livekit-client: + specifier: ^1.12.0 || ^2.0.0 + version: 2.0.8 devDependencies: '@changesets/cli': @@ -46,9 +49,6 @@ devDependencies: eslint-plugin-import: specifier: 2.27.5 version: 2.27.5(eslint@8.39.0) - livekit-client: - specifier: ^1.12.0 - version: 1.14.4 prettier: specifier: ^2.8.8 version: 2.8.8 @@ -201,9 +201,9 @@ packages: to-fast-properties: 2.0.0 dev: true - /@bufbuild/protobuf@1.4.2: - resolution: {integrity: sha512-JyEH8Z+OD5Sc2opSg86qMHn1EM1Sa+zj/Tc0ovxdwk56ByVNONJSabuCUbLQp+eKN3rWNfrho0X+3SEqEPXIow==} - dev: true + /@bufbuild/protobuf@1.8.0: + resolution: {integrity: sha512-qR9FwI8QKIveDnUYutvfzbC21UZJJryYrLuZGjeZ/VGz+vXelUkK+xgkOHsvPEdYEdxtgUUq4313N8QtOehJ1Q==} + dev: false /@changesets/apply-release-plan@6.1.4: resolution: {integrity: sha512-FMpKF1fRlJyCZVYHr3CbinpZZ+6MwvOtWUuO8uo+svcATEoc1zRDcj23pAurJ2TZ/uVz1wFHH6K3NlACy0PLew==} @@ -693,6 +693,12 @@ packages: - encoding dev: true + /@livekit/protocol@1.10.4: + resolution: {integrity: sha512-7Occ6l5VqjsKsUjpLeVJXJkip4ce22iG9QR/haOkrDXYwGSY+TwCA4ubbJ81aQBt2z0kHVZuaQWdzJSDLI+Kag==} + dependencies: + '@bufbuild/protobuf': 1.8.0 + dev: false + /@manypkg/find-root@1.1.0: resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} dependencies: @@ -1758,7 +1764,7 @@ packages: /events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} - dev: true + dev: false /execa@5.1.1: resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} @@ -2418,17 +2424,18 @@ packages: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} dev: true - /livekit-client@1.14.4: - resolution: {integrity: sha512-sXNH+qhqkmXeCVsw3hcEbUsh9QfguBIGgM3glM0v0+YW4C2LdeGWD4pQBLR7USbkwbbVMEHDmqr/iY6XvEIoeQ==} + /livekit-client@2.0.8: + resolution: {integrity: sha512-d4x+OgJR/CcwsaGwYHohaCVwZVBk+urb1xZw48iLSbhyKlN39J2NQkHGiHro+l8b3SwMnyP3IWkOwS/dxrWz2g==} dependencies: - '@bufbuild/protobuf': 1.4.2 + '@livekit/protocol': 1.10.4 events: 3.3.0 - loglevel: 1.8.1 - sdp-transform: 2.14.1 + loglevel: 1.9.1 + sdp-transform: 2.14.2 ts-debounce: 4.0.0 + tslib: 2.6.2 typed-emitter: 2.1.0 webrtc-adapter: 8.2.3 - dev: true + dev: false /load-tsconfig@0.2.5: resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} @@ -2475,10 +2482,10 @@ packages: resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} dev: true - /loglevel@1.8.1: - resolution: {integrity: sha512-tCRIJM51SHjAayKwC+QAg8hT8vg6z7GSgLJKGvzuPb1Wc+hLzqtuVLxp6/HzSPOozuK+8ErAhy7U/sVzw8Dgfg==} + /loglevel@1.9.1: + resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==} engines: {node: '>= 0.6.0'} - dev: true + dev: false /lru-cache@4.1.5: resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==} @@ -3008,7 +3015,7 @@ packages: requiresBuild: true dependencies: tslib: 2.6.2 - dev: true + dev: false optional: true /safe-array-concat@1.0.1: @@ -3033,14 +3040,14 @@ packages: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} dev: true - /sdp-transform@2.14.1: - resolution: {integrity: sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==} + /sdp-transform@2.14.2: + resolution: {integrity: sha512-icY6jVao7MfKCieyo1AyxFYm1baiM+fA00qW/KrNNVlkxHAd34riEKuEkUe4bBb3gJwLJZM+xT60Yj1QL8rHiA==} hasBin: true - dev: true + dev: false /sdp@3.2.0: resolution: {integrity: sha512-d7wDPgDV3DDiqulJjKiV2865wKsJ34YI+NDREbm+FySq6WuKOikwyNQcm+doLAZ1O6ltdO0SeKle2xMpN3Brgw==} - dev: true + dev: false /semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} @@ -3352,7 +3359,7 @@ packages: /ts-debounce@4.0.0: resolution: {integrity: sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg==} - dev: true + dev: false /ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} @@ -3373,9 +3380,7 @@ packages: /tslib@2.6.2: resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} - requiresBuild: true - dev: true - optional: true + dev: false /tsup@7.2.0(typescript@5.2.2): resolution: {integrity: sha512-vDHlczXbgUvY3rWvqFEbSqmC1L7woozbzngMqTtL2PGBODTtWlRwGDDawhvWzr5c1QjKe4OAKqJGfE1xeXUvtQ==} @@ -3506,7 +3511,7 @@ packages: resolution: {integrity: sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==} optionalDependencies: rxjs: 7.8.1 - dev: true + dev: false /typescript@5.2.2: resolution: {integrity: sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==} @@ -3606,7 +3611,7 @@ packages: engines: {node: '>=6.0.0', npm: '>=3.10.0'} dependencies: sdp: 3.2.0 - dev: true + dev: false /whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} diff --git a/src/transformers/BackgroundTransformer.ts b/src/transformers/BackgroundTransformer.ts index dd91536..9c610e9 100644 --- a/src/transformers/BackgroundTransformer.ts +++ b/src/transformers/BackgroundTransformer.ts @@ -8,7 +8,10 @@ export type SegmenterOptions = Partial { @@ -36,12 +39,14 @@ export default class BackgroundProcessor extends VideoTransformer