diff --git a/.changeset/wise-doors-jump.md b/.changeset/wise-doors-jump.md new file mode 100644 index 000000000..21e79827f --- /dev/null +++ b/.changeset/wise-doors-jump.md @@ -0,0 +1,6 @@ +--- +"@livekit/components-core": patch +"@livekit/components-react": patch +--- + +Add krisp hook diff --git a/packages/core/src/components/trackMutedIndicator.ts b/packages/core/src/components/trackMutedIndicator.ts index e8df284b5..71ab051c1 100644 --- a/packages/core/src/components/trackMutedIndicator.ts +++ b/packages/core/src/components/trackMutedIndicator.ts @@ -1,3 +1,4 @@ +// @ts-ignore some module resolutions (other than 'node') choke on this import type { Styles } from '@livekit/components-styles/dist/types_unprefixed/index.scss'; import { Track } from 'livekit-client'; import { mutedObserver } from '../observables/participant'; diff --git a/packages/core/src/observables/participant.ts b/packages/core/src/observables/participant.ts index 901c7791a..0af8da1b8 100644 --- a/packages/core/src/observables/participant.ts +++ b/packages/core/src/observables/participant.ts @@ -1,6 +1,7 @@ import type { ParticipantPermission } from '@livekit/protocol'; import { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client'; import { ParticipantEvent, RoomEvent, Track } from 'livekit-client'; +// @ts-ignore some module resolutions (other than 'node') choke on this import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant'; import type { Subscriber } from 'rxjs'; import { Observable, map, startWith, switchMap } from 'rxjs'; diff --git a/packages/core/src/observables/room.ts b/packages/core/src/observables/room.ts index d53d05756..1094f35d7 100644 --- a/packages/core/src/observables/room.ts +++ b/packages/core/src/observables/room.ts @@ -2,6 +2,7 @@ import type { Subscriber, Subscription } from 'rxjs'; import { Subject, map, Observable, startWith, finalize, filter, concat } from 'rxjs'; import type { Participant, TrackPublication } from 'livekit-client'; import { LocalParticipant, Room, RoomEvent, Track } from 'livekit-client'; +// @ts-ignore some module resolutions (other than 'node') choke on this import type { RoomEventCallbacks } from 'livekit-client/dist/src/room/Room'; import { log } from '../logger'; export function observeRoomEvents(room: Room, ...events: RoomEvent[]): Observable { diff --git a/packages/core/src/observables/track.ts b/packages/core/src/observables/track.ts index b55c52b9d..0b5b895d5 100644 --- a/packages/core/src/observables/track.ts +++ b/packages/core/src/observables/track.ts @@ -14,6 +14,7 @@ import type { TrackReference } from '../track-reference'; import { observeRoomEvents } from './room'; import type { ParticipantTrackIdentifier } from '../types'; import { observeParticipantEvents } from './participant'; +// @ts-ignore some module resolutions (other than 'node') choke on this import type { PublicationEventCallbacks } from 'livekit-client/dist/src/room/track/TrackPublication'; export function trackObservable(track: TrackPublication) { diff --git a/packages/react/.size-limit.js b/packages/react/.size-limit.js index 4bf756865..bae305efa 100644 --- a/packages/react/.size-limit.js +++ b/packages/react/.size-limit.js @@ -4,20 +4,20 @@ module.exports = [ path: 'dist/index.mjs', import: '{ LiveKitRoom }', limit: '4 kB', - ignore: ['livekit-client', 'react', 'react-dom', 'loglevel'], + ignore: ['livekit-client', 'react', 'react-dom', 'loglevel', '@livekit/krisp-noise-filter'], }, { name: 'LiveKitRoom with VideoConference', path: 'dist/index.mjs', import: '{ LiveKitRoom, VideoConference }', limit: '40 kB', - ignore: ['livekit-client', 'react', 'react-dom', 'loglevel'], + ignore: ['livekit-client', 'react', 'react-dom', 'loglevel', '@livekit/krisp-noise-filter'], }, { name: 'All exports', path: 'dist/index.mjs', import: '*', limit: '100 kB', - ignore: ['livekit-client', 'react', 'react-dom', 'loglevel'], + ignore: ['livekit-client', 'react', 'react-dom', 'loglevel', '@livekit/krisp-noise-filter'], }, ]; diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md index 91c189ce7..78e47c98b 100644 --- a/packages/react/etc/components-react.api.md +++ b/packages/react/etc/components-react.api.md @@ -12,12 +12,14 @@ import { ConnectionState as ConnectionState_2 } from 'livekit-client'; import type { CreateLocalTracksOptions } from 'livekit-client'; import type { DataPublishOptions } from 'livekit-client'; import type { HTMLAttributes } from 'react'; +import type { KrispNoiseFilterProcessor } from '@livekit/krisp-noise-filter'; import { LocalAudioTrack } from 'livekit-client'; import { LocalParticipant } from 'livekit-client'; import type { LocalTrack } from 'livekit-client'; import { LocalTrackPublication } from 'livekit-client'; import { LocalVideoTrack } from 'livekit-client'; import type { MediaDeviceFailure } from 'livekit-client'; +import type { NoiseFilterOptions } from '@livekit/krisp-noise-filter'; import { Participant } from 'livekit-client'; import type { ParticipantEvent } from 'livekit-client'; import type { ParticipantKind } from 'livekit-client'; @@ -883,6 +885,21 @@ export function useIsRecording(room?: Room): boolean; // @public export function useIsSpeaking(participant?: Participant): boolean; +// @alpha +export function useKrispNoiseFilter(options?: useKrispNoiseFilterOptions): { + setNoiseFilterEnabled: (enable: boolean) => Promise; + isNoiseFilterEnabled: boolean; + isNoiseFilterPending: boolean; + processor: KrispNoiseFilterProcessor | undefined; +}; + +// @alpha (undocumented) +export interface useKrispNoiseFilterOptions { + // (undocumented) + filterOptions?: NoiseFilterOptions; + trackRef?: TrackReferenceOrPlaceholder; +} + // @public export function useLayoutContext(): LayoutContextType; diff --git a/packages/react/package.json b/packages/react/package.json index 52895e6e5..c3a08d02b 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -52,11 +52,17 @@ "usehooks-ts": "3.1.0" }, "peerDependencies": { + "@livekit/krisp-noise-filter": "^0.2.12", "livekit-client": "^2.5.4", "react": ">=18", "react-dom": ">=18", "tslib": "^2.6.2" }, + "peerDependenciesMeta": { + "@livekit/krisp-noise-filter": { + "optional": true + } + }, "devDependencies": { "@livekit/protocol": "^1.22.0", "@microsoft/api-extractor": "^7.35.0", diff --git a/packages/react/src/hooks/cloud/krisp/useKrispNoiseFilter.ts b/packages/react/src/hooks/cloud/krisp/useKrispNoiseFilter.ts new file mode 100644 index 000000000..5e05ab88f --- /dev/null +++ b/packages/react/src/hooks/cloud/krisp/useKrispNoiseFilter.ts @@ -0,0 +1,104 @@ +import * as React from 'react'; +import { LocalAudioTrack } from 'livekit-client'; +import type { KrispNoiseFilterProcessor, NoiseFilterOptions } from '@livekit/krisp-noise-filter'; +import type { TrackReferenceOrPlaceholder } from '@livekit/components-core'; +import { useLocalParticipant } from '../../useLocalParticipant'; + +/** + * @alpha + */ +export interface useKrispNoiseFilterOptions { + /** + * by default the hook will use the localParticipant's microphone track publication. + * You can override this behavior by passing in a target TrackReference here + */ + trackRef?: TrackReferenceOrPlaceholder; + filterOptions?: NoiseFilterOptions; +} + +/** + * This hook is a convenience helper for enabling Krisp Enhanced Audio Noise Cancellation on LiveKit audio tracks. + * It returns a `setNoiseFilterEnabled` method to conveniently toggle between enabled and disabled states. + * + * @remarks Krisp noise filter is a feature that's only supported on LiveKit cloud plans + * @alpha + * @example + * ```tsx + * const krisp = useKrispNoiseFilter(); + * return krisp.setNoiseFilterEnabled(ev.target.checked)} + checked={krisp.isNoiseFilterEnabled} + disabled={krisp.isNoiseFilterPending} + /> + * ``` + */ +export function useKrispNoiseFilter(options: useKrispNoiseFilterOptions = {}) { + const [shouldEnable, setShouldEnable] = React.useState(false); + const [isNoiseFilterPending, setIsNoiseFilterPending] = React.useState(false); + const [isNoiseFilterEnabled, setIsNoiseFilterEnabled] = React.useState(false); + let micPublication = useLocalParticipant().microphoneTrack; + const [krispProcessor, setKrispProcessor] = React.useState< + KrispNoiseFilterProcessor | undefined + >(); + if (options.trackRef) { + micPublication = options.trackRef.publication; + } + + const setNoiseFilterEnabled = React.useCallback(async (enable: boolean) => { + if (enable) { + const { KrispNoiseFilter, isKrispNoiseFilterSupported } = await import( + '@livekit/krisp-noise-filter' + ); + + if (!isKrispNoiseFilterSupported()) { + console.warn('Krisp noise filter is not supported in this browser'); + return; + } + if (!krispProcessor) { + setKrispProcessor(KrispNoiseFilter(options.filterOptions)); + } + } + setShouldEnable((prev) => { + if (prev !== enable) { + setIsNoiseFilterPending(true); + } + return enable; + }); + }, []); + + React.useEffect(() => { + if (micPublication && micPublication.track instanceof LocalAudioTrack && krispProcessor) { + const currentProcessor = micPublication.track.getProcessor(); + if (currentProcessor && currentProcessor.name === 'livekit-noise-filter') { + setIsNoiseFilterPending(true); + (currentProcessor as KrispNoiseFilterProcessor).setEnabled(shouldEnable).finally(() => { + setIsNoiseFilterPending(false); + setIsNoiseFilterEnabled(shouldEnable); + }); + } else if (!currentProcessor && shouldEnable) { + setIsNoiseFilterPending(true); + micPublication?.track + ?.setProcessor(krispProcessor) + .then(() => krispProcessor.setEnabled(shouldEnable)) + .then(() => { + setIsNoiseFilterEnabled(true); + }) + .catch((e: any) => { + setIsNoiseFilterEnabled(false); + console.error(e); + }) + .finally(() => { + setIsNoiseFilterPending(false); + }); + } + } + }, [shouldEnable, micPublication, krispProcessor]); + + return { + setNoiseFilterEnabled, + isNoiseFilterEnabled, + isNoiseFilterPending, + processor: krispProcessor, + }; +} diff --git a/packages/react/src/hooks/index.ts b/packages/react/src/hooks/index.ts index 1c798c164..daa7a6a83 100644 --- a/packages/react/src/hooks/index.ts +++ b/packages/react/src/hooks/index.ts @@ -54,3 +54,4 @@ export * from './useTrackTranscription'; export * from './useVoiceAssistant'; export * from './useParticipantAttributes'; export * from './useIsRecording'; +export { useKrispNoiseFilter, useKrispNoiseFilterOptions } from './cloud/krisp/useKrispNoiseFilter'; diff --git a/packages/react/src/hooks/useParticipantAttributes.ts b/packages/react/src/hooks/useParticipantAttributes.ts index a2b62f5e3..123089fbb 100644 --- a/packages/react/src/hooks/useParticipantAttributes.ts +++ b/packages/react/src/hooks/useParticipantAttributes.ts @@ -57,7 +57,7 @@ export function useParticipantAttribute( } const subscription = participantAttributesObserver(p).subscribe((val) => { if (val.changed[attributeKey] !== undefined) { - setAttribute(val.changed[attributeKey]); + setAttribute(val.attributes[attributeKey]); } }); return () => { diff --git a/packages/react/tsup.config.ts b/packages/react/tsup.config.ts index 6504dde56..5e39ee06d 100644 --- a/packages/react/tsup.config.ts +++ b/packages/react/tsup.config.ts @@ -5,5 +5,5 @@ import defaults from '../../tsup.config'; export default defineConfig({ ...defaults, entry: ['src/index.ts', 'src/hooks/index.ts', 'src/prefabs/index.ts'], - external: ['livekit-client', 'react', 'react-dom'], + external: ['livekit-client', 'react', 'react-dom', '@livekit/krisp-noise-filter'], }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 321d5d11e..569585b2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -210,6 +210,9 @@ importers: '@livekit/components-core': specifier: workspace:* version: link:../core + '@livekit/krisp-noise-filter': + specifier: ^0.2.12 + version: 0.2.12(livekit-client@2.5.4) clsx: specifier: 2.1.1 version: 2.1.1 @@ -1618,6 +1621,11 @@ packages: '@livekit/changesets-changelog-github@0.0.4': resolution: {integrity: sha512-MXaiLYwgkYciZb8G2wkVtZ1pJJzZmVx5cM30Q+ClslrIYyAqQhRbPmZDM79/5CGxb1MTemR/tfOM25tgJgAK0g==} + '@livekit/krisp-noise-filter@0.2.12': + resolution: {integrity: sha512-z7qSa3A6fn/DYTt0rITNAK0sNpBTzlnb29aM0ks8UfpbfTnnjAaFv3AC695mUq9iICPKrd5jOQT71gowiQ+Otg==} + peerDependencies: + livekit-client: ^2.0.8 + '@livekit/protocol@1.21.0': resolution: {integrity: sha512-3TohFPNZy1axTuoDLU6mA1rwuP4VawgehvX52OoLJnU+fNQYfmMJqz8k7NSh79jG5I8Og77YYhT905Omrhli2A==} @@ -9066,6 +9074,10 @@ snapshots: transitivePeerDependencies: - encoding + '@livekit/krisp-noise-filter@0.2.12(livekit-client@2.5.4)': + dependencies: + livekit-client: 2.5.4 + '@livekit/protocol@1.21.0': dependencies: '@bufbuild/protobuf': 1.10.0