From 9a59311d31602cb98ebab0c6911b9dffaaf03942 Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 10 Apr 2024 14:45:13 +0200 Subject: [PATCH] Signal local audio track feature updates (#1087) * Signal local audio track feature updates * Create gold-tomatoes-protect.md --- .changeset/gold-tomatoes-protect.md | 5 +++ pnpm-lock.yaml | 6 ++-- src/api/SignalClient.ts | 9 ++++++ src/room/events.ts | 5 +++ src/room/participant/LocalParticipant.ts | 14 +++++++++ src/room/track/LocalAudioTrack.ts | 40 ++++++++++++++++++++++++ src/room/track/LocalTrackPublication.ts | 30 ++++++++++++++++-- src/room/track/Track.ts | 2 ++ 8 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 .changeset/gold-tomatoes-protect.md diff --git a/.changeset/gold-tomatoes-protect.md b/.changeset/gold-tomatoes-protect.md new file mode 100644 index 0000000000..ddcb11668f --- /dev/null +++ b/.changeset/gold-tomatoes-protect.md @@ -0,0 +1,5 @@ +--- +"livekit-client": patch +--- + +Signal local audio track feature updates diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ea2d3965e5..87c0647511 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3311,7 +3311,7 @@ packages: dependencies: semver: 7.6.0 shelljs: 0.8.5 - typescript: 5.5.0-dev.20240404 + typescript: 5.5.0-dev.20240410 dev: true /electron-to-chromium@1.4.724: @@ -6137,8 +6137,8 @@ packages: hasBin: true dev: true - /typescript@5.5.0-dev.20240404: - resolution: {integrity: sha512-Knb9Yx0JJHc0mmqXLEPPKNSwOvPQrtYZEDLQY7Wns7LckkQl82AZ+OGTPG/ofwqx2QeDCHCtKjutZPy1UEiwKA==} + /typescript@5.5.0-dev.20240410: + resolution: {integrity: sha512-OvGiFb8iPBHHqR8RuhIeCM+j2W1TtPbTZLuesCEi4gulAxzkOP2B3jDPTWmcOmxvUeJN5pNzlKoi/reRn1BZww==} engines: {node: '>=14.17'} hasBin: true dev: true diff --git a/src/api/SignalClient.ts b/src/api/SignalClient.ts index 13b9a4c000..0dad49d63f 100644 --- a/src/api/SignalClient.ts +++ b/src/api/SignalClient.ts @@ -1,5 +1,6 @@ import { AddTrackRequest, + AudioTrackFeature, ClientInfo, ConnectionQualityUpdate, DisconnectReason, @@ -27,6 +28,7 @@ import { TrackPublishedResponse, TrackUnpublishedResponse, TrickleRequest, + UpdateLocalAudioTrack, UpdateParticipantMetadata, UpdateSubscription, UpdateTrackSettings, @@ -583,6 +585,13 @@ export class SignalClient { ]); } + sendUpdateLocalAudioTrack(trackSid: string, features: AudioTrackFeature[]) { + return this.sendRequest({ + case: 'updateAudioTrack', + value: new UpdateLocalAudioTrack({ trackSid, features }), + }); + } + sendLeave() { return this.sendRequest({ case: 'leave', diff --git a/src/room/events.ts b/src/room/events.ts index cc8cc614b7..2510e7be5d 100644 --- a/src/room/events.ts +++ b/src/room/events.ts @@ -557,4 +557,9 @@ export enum TrackEvent { * @internal */ TrackProcessorUpdate = 'trackProcessorUpdate', + + /** + * @internal + */ + AudioTrackFeatureUpdate = 'audioTrackFeatureUpdate', } diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index dc59c89670..cdb5b4a5cf 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -685,6 +685,7 @@ export default class LocalParticipant extends Participant { track.on(TrackEvent.Ended, this.handleTrackEnded); track.on(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused); track.on(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed); + track.on(TrackEvent.AudioTrackFeatureUpdate, this.onTrackFeatureUpdate); // create track publication from track const req = new AddTrackRequest({ @@ -1037,6 +1038,7 @@ export default class LocalParticipant extends Participant { track.off(TrackEvent.Ended, this.handleTrackEnded); track.off(TrackEvent.UpstreamPaused, this.onTrackUpstreamPaused); track.off(TrackEvent.UpstreamResumed, this.onTrackUpstreamResumed); + track.off(TrackEvent.AudioTrackFeatureUpdate, this.onTrackFeatureUpdate); if (stopOnUnpublish === undefined) { stopOnUnpublish = this.roomOptions?.stopLocalTrackOnUnpublish ?? true; @@ -1293,6 +1295,18 @@ export default class LocalParticipant extends Participant { this.onTrackMuted(track, track.isMuted); }; + private onTrackFeatureUpdate = (track: LocalAudioTrack) => { + const pub = this.audioTrackPublications.get(track.sid!); + if (!pub) { + this.log.warn( + `Could not update local audio track settings, missing publication for track ${track.sid}`, + this.logContext, + ); + return; + } + this.engine.client.sendUpdateLocalAudioTrack(pub.trackSid, pub.getTrackFeatures()); + }; + private handleSubscribedQualityUpdate = async (update: SubscribedQualityUpdate) => { if (!this.roomOptions?.dynacast) { return; diff --git a/src/room/track/LocalAudioTrack.ts b/src/room/track/LocalAudioTrack.ts index a21c603e9f..020705f664 100644 --- a/src/room/track/LocalAudioTrack.ts +++ b/src/room/track/LocalAudioTrack.ts @@ -1,3 +1,4 @@ +import { AudioTrackFeature } from '@livekit/protocol'; import { TrackEvent } from '../events'; import { computeBitrate, monitorFrequency } from '../stats'; import type { AudioSenderStats } from '../stats'; @@ -15,8 +16,17 @@ export default class LocalAudioTrack extends LocalTrack { private prevStats?: AudioSenderStats; + private isKrispNoiseFilterEnabled = false; + protected processor?: TrackProcessor | undefined; + /** + * boolean indicating whether enhanced noise cancellation is currently being used on this track + */ + get enhancedNoiseCancellation() { + return this.isKrispNoiseFilterEnabled; + } + /** * * @param mediaTrack @@ -152,6 +162,28 @@ export default class LocalAudioTrack extends LocalTrack { this.prevStats = stats; }; + private handleKrispNoiseFilterEnable = () => { + this.isKrispNoiseFilterEnabled = true; + this.log.debug(`Krisp noise filter enabled`, this.logContext); + this.emit( + TrackEvent.AudioTrackFeatureUpdate, + this, + AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION, + true, + ); + }; + + private handleKrispNoiseFilterDisable = () => { + this.isKrispNoiseFilterEnabled = false; + this.log.debug(`Krisp noise filter disabled`, this.logContext); + this.emit( + TrackEvent.AudioTrackFeatureUpdate, + this, + AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION, + false, + ); + }; + async setProcessor(processor: TrackProcessor) { const unlock = await this.processorLock.lock(); try { @@ -175,6 +207,14 @@ export default class LocalAudioTrack extends LocalTrack { this.processor = processor; if (this.processor.processedTrack) { await this.sender?.replaceTrack(this.processor.processedTrack); + this.processor.processedTrack.addEventListener( + 'enable-lk-krisp-noise-filter', + this.handleKrispNoiseFilterEnable, + ); + this.processor.processedTrack.addEventListener( + 'disable-lk-krisp-noise-filter', + this.handleKrispNoiseFilterDisable, + ); } this.emit(TrackEvent.TrackProcessorUpdate, this.processor); } finally { diff --git a/src/room/track/LocalTrackPublication.ts b/src/room/track/LocalTrackPublication.ts index 14beece3cb..6165b09391 100644 --- a/src/room/track/LocalTrackPublication.ts +++ b/src/room/track/LocalTrackPublication.ts @@ -1,7 +1,7 @@ -import type { TrackInfo } from '@livekit/protocol'; +import { AudioTrackFeature, TrackInfo } from '@livekit/protocol'; import { TrackEvent } from '../events'; import type { LoggerOptions } from '../types'; -import type LocalAudioTrack from './LocalAudioTrack'; +import LocalAudioTrack from './LocalAudioTrack'; import type LocalTrack from './LocalTrack'; import type LocalVideoTrack from './LocalVideoTrack'; import type { Track } from './Track'; @@ -82,6 +82,32 @@ export default class LocalTrackPublication extends TrackPublication { await this.track?.resumeUpstream(); } + getTrackFeatures() { + if (this.track instanceof LocalAudioTrack) { + const settings = this.track!.mediaStreamTrack.getSettings(); + const features: Set = new Set(); + if (settings.autoGainControl) { + features.add(AudioTrackFeature.TF_AUTO_GAIN_CONTROL); + } + if (settings.echoCancellation) { + features.add(AudioTrackFeature.TF_ECHO_CANCELLATION); + } + if (settings.noiseSuppression) { + features.add(AudioTrackFeature.TF_NOISE_SUPPRESSION); + } + if (settings.channelCount && settings.channelCount > 1) { + features.add(AudioTrackFeature.TF_STEREO); + } + if (!this.options?.dtx) { + features.add(AudioTrackFeature.TF_STEREO); + } + if (this.track.enhancedNoiseCancellation) { + features.add(AudioTrackFeature.TF_ENHANCED_NOISE_CANCELLATION); + } + return Array.from(features.values()); + } else return []; + } + handleTrackEnded = () => { this.emit(TrackEvent.Ended); }; diff --git a/src/room/track/Track.ts b/src/room/track/Track.ts index 0c826710d3..59024d7dfc 100644 --- a/src/room/track/Track.ts +++ b/src/room/track/Track.ts @@ -1,4 +1,5 @@ import { + AudioTrackFeature, VideoQuality as ProtoQuality, StreamState as ProtoStreamState, TrackSource, @@ -504,4 +505,5 @@ export type TrackEventCallbacks = { upstreamPaused: (track: any) => void; upstreamResumed: (track: any) => void; trackProcessorUpdate: (processor?: TrackProcessor) => void; + audioTrackFeatureUpdate: (track: any, feature: AudioTrackFeature, enabled: boolean) => void; };