From d35dffd16131cac43279300071c595f30981767f Mon Sep 17 00:00:00 2001 From: lukasIO Date: Wed, 14 Aug 2024 09:13:25 +0200 Subject: [PATCH] Add useVoiceAssistant (#917) --- .changeset/dull-pots-applaud.md | 7 +++ packages/core/etc/components-core.api.md | 19 +++--- packages/core/src/observables/participant.ts | 20 +++++- packages/core/src/observables/room.ts | 10 +-- packages/react/etc/components-react.api.md | 42 +++++++++++-- packages/react/src/components/RoomName.tsx | 9 +-- .../participant/ParticipantTile.tsx | 3 +- packages/react/src/context/index.ts | 1 + .../src/context/voice-assistant-context.ts | 5 ++ packages/react/src/hooks/index.ts | 1 + packages/react/src/hooks/useIsEncrypted.ts | 3 +- .../src/hooks/useParticipantAttributes.ts | 16 +++-- .../react/src/hooks/useParticipantInfo.ts | 13 ++-- packages/react/src/hooks/useTrackSyncTime.ts | 8 +-- .../react/src/hooks/useTrackTranscription.ts | 6 +- packages/react/src/hooks/useVoiceAssistant.ts | 63 +++++++++++++++++++ 16 files changed, 181 insertions(+), 45 deletions(-) create mode 100644 .changeset/dull-pots-applaud.md create mode 100644 packages/react/src/context/voice-assistant-context.ts create mode 100644 packages/react/src/hooks/useVoiceAssistant.ts diff --git a/.changeset/dull-pots-applaud.md b/.changeset/dull-pots-applaud.md new file mode 100644 index 000000000..8b915ef04 --- /dev/null +++ b/.changeset/dull-pots-applaud.md @@ -0,0 +1,7 @@ +--- +"@livekit/components-core": patch +"@livekit/components-react": patch +"eslint-config-lk-custom": patch +--- + +Add useVoiceAssistant diff --git a/packages/core/etc/components-core.api.md b/packages/core/etc/components-core.api.md index 8160da519..9d3a13605 100644 --- a/packages/core/etc/components-core.api.md +++ b/packages/core/etc/components-core.api.md @@ -15,7 +15,7 @@ import type { LocalParticipant } from 'livekit-client'; import { LocalVideoTrack } from 'livekit-client'; import loglevel from 'loglevel'; import { Observable } from 'rxjs'; -import type { Participant } from 'livekit-client'; +import { Participant } from 'livekit-client'; import { ParticipantEvent } from 'livekit-client'; import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant'; import type { ParticipantKind } from 'livekit-client'; @@ -174,7 +174,7 @@ export const defaultUserChoices: LocalUserChoices; export function didActiveSegmentsChange(prevActive: T[], newActive: T[]): boolean; // @public (undocumented) -export function encryptionStatusObservable(room: Room, participant: Participant): Observable; +export function encryptionStatusObservable(room: Room, participant: Participant | undefined): Observable; // @public (undocumented) export function getActiveTranscriptionSegments(segments: ReceivedTranscriptionSegment[], syncTimes: { @@ -312,9 +312,12 @@ export function observeTrackEvents(track: TrackPublication, ...events: TrackEven export function participantAttributesObserver(participant: Participant): Observable<{ changed: Readonly>; attributes: Readonly>; -} | { - changed: Readonly>; - attributes: Readonly>; +}>; + +// @public (undocumented) +export function participantAttributesObserver(participant: undefined): Observable<{ + changed: undefined; + attributes: undefined; }>; // Warning: (ae-incompatible-release-tags) The symbol "participantByIdentifierObserver" is marked as @public, but its signature references "ParticipantIdentifier" which is marked as @beta @@ -347,7 +350,7 @@ export type ParticipantIdentifier = RequireAtLeastOne<{ }, 'identity' | 'kind'>; // @public (undocumented) -export function participantInfoObserver(participant: Participant): Observable<{ +export function participantInfoObserver(participant?: Participant): Observable<{ name: string | undefined; identity: string; metadata: string | undefined; @@ -355,7 +358,7 @@ export function participantInfoObserver(participant: Participant): Observable<{ name: string | undefined; identity: string; metadata: string | undefined; -}>; +}> | undefined; // @public (undocumented) export interface ParticipantMedia { @@ -572,7 +575,7 @@ export function setupParticipantName(participant: Participant): { name: string | undefined; identity: string; metadata: string | undefined; - }>; + }> | undefined; }; // @public (undocumented) diff --git a/packages/core/src/observables/participant.ts b/packages/core/src/observables/participant.ts index b80d6d35c..901c7791a 100644 --- a/packages/core/src/observables/participant.ts +++ b/packages/core/src/observables/participant.ts @@ -1,5 +1,5 @@ import type { ParticipantPermission } from '@livekit/protocol'; -import type { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client'; +import { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client'; import { ParticipantEvent, RoomEvent, Track } from 'livekit-client'; import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant'; import type { Subscriber } from 'rxjs'; @@ -85,7 +85,10 @@ export function createTrackObserver(participant: Participant, options: TrackIden ); } -export function participantInfoObserver(participant: Participant) { +export function participantInfoObserver(participant?: Participant) { + if (!participant) { + return undefined; + } const observer = observeParticipantEvents( participant, ParticipantEvent.ParticipantMetadataChanged, @@ -287,7 +290,18 @@ export function participantByIdentifierObserver( return observable; } -export function participantAttributesObserver(participant: Participant) { +export function participantAttributesObserver(participant: Participant): Observable<{ + changed: Readonly>; + attributes: Readonly>; +}>; +export function participantAttributesObserver(participant: undefined): Observable<{ + changed: undefined; + attributes: undefined; +}>; +export function participantAttributesObserver(participant: Participant | undefined) { + if (typeof participant === 'undefined') { + return new Observable<{ changed: undefined; attributes: undefined }>(); + } return participantEventSelector(participant, ParticipantEvent.AttributesChanged).pipe( map(([changedAttributes]) => { return { diff --git a/packages/core/src/observables/room.ts b/packages/core/src/observables/room.ts index 4cd5401b3..c3f1fc166 100644 --- a/packages/core/src/observables/room.ts +++ b/packages/core/src/observables/room.ts @@ -248,16 +248,18 @@ export function createActiveDeviceObservable(room: Room, kind: MediaDeviceKind) ); } -export function encryptionStatusObservable(room: Room, participant: Participant) { +export function encryptionStatusObservable(room: Room, participant: Participant | undefined) { return roomEventSelector(room, RoomEvent.ParticipantEncryptionStatusChanged).pipe( filter( ([, p]) => - participant.identity === p?.identity || - (!p && participant.identity === room.localParticipant.identity), + participant?.identity === p?.identity || + (!p && participant?.identity === room.localParticipant.identity), ), map(([encrypted]) => encrypted), startWith( - participant instanceof LocalParticipant ? participant.isE2EEEnabled : participant.isEncrypted, + participant instanceof LocalParticipant + ? participant.isE2EEEnabled + : !!participant?.isEncrypted, ), ); } diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md index 2839302a7..f0fefc3cd 100644 --- a/packages/react/etc/components-react.api.md +++ b/packages/react/etc/components-react.api.md @@ -573,7 +573,7 @@ export interface RoomAudioRendererProps { export const RoomContext: React_2.Context; // @public -export const RoomName: (props: RoomNameProps & React_2.RefAttributes) => React_2.ReactNode; +export const RoomName: React_2.FC>; // @public (undocumented) export interface RoomNameProps extends React_2.HTMLAttributes { @@ -637,6 +637,13 @@ export interface TrackMutedIndicatorProps extends React_2.HTMLAttributes; +// Warning: (ae-internal-missing-underscore) The name "TrackRefContextIfNeeded" should be prefixed with an underscore because the declaration is marked as @internal +// +// @internal +export function TrackRefContextIfNeeded(props: React_2.PropsWithChildren<{ + trackRef?: TrackReferenceOrPlaceholder; +}>): React_2.JSX.Element; + // @public (undocumented) export type TrackReference = { participant: Participant; @@ -881,7 +888,7 @@ export function useMediaDeviceSelect({ kind, room, track, requestPermissions, on devices: MediaDeviceInfo[]; className: string; activeDeviceId: string; - setActiveMediaDevice: (id: string, options?: SetMediaDeviceOptions) => Promise; + setActiveMediaDevice: (id: string, options?: SetMediaDeviceOptions | undefined) => Promise; }; // @public (undocumented) @@ -916,7 +923,7 @@ export function useParticipantAttribute(attributeKey: string, options?: UseParti // @public (undocumented) export function useParticipantAttributes(props?: UseParticipantAttributesOptions): { - attributes: Readonly>; + attributes: Readonly> | undefined; }; // @public @@ -930,7 +937,7 @@ export function useParticipantContext(): Participant; // @public (undocumented) export function useParticipantInfo(props?: UseParticipantInfoOptions): { - identity: string; + identity: string | undefined; name: string | undefined; metadata: string | undefined; }; @@ -1152,7 +1159,7 @@ export type UseTracksOptions = { // @public export function useTrackToggle({ source, onChange, initialState, captureOptions, publishOptions, onDeviceError, ...rest }: UseTrackToggleProps): { - toggle: (forceState?: boolean, captureOptions?: CaptureOptionsBySource | undefined) => Promise; + toggle: (forceState?: boolean | undefined, captureOptions?: CaptureOptionsBySource | undefined) => Promise; enabled: boolean; pending: boolean; track: LocalTrackPublication | undefined; @@ -1164,7 +1171,7 @@ export interface UseTrackToggleProps extends Omit TrackReferenceOrPlaceholder[]; } +// @alpha +export function useVoiceAssistant(): VoiceAssistant; + // @public export function VideoConference({ chatMessageFormatter, chatMessageDecoder, chatMessageEncoder, SettingsComponent, ...props }: VideoConferenceProps): React_2.JSX.Element; @@ -1211,6 +1221,26 @@ export interface VideoTrackProps extends React_2.VideoHTMLAttributes; + +// @alpha (undocumented) +export type VoiceAssistantState = 'disconnected' | 'connecting' | 'listening' | 'thinking' | 'speaking'; + // @public (undocumented) export type WidgetState = { showChat: boolean; diff --git a/packages/react/src/components/RoomName.tsx b/packages/react/src/components/RoomName.tsx index f5642f5ca..893c52b07 100644 --- a/packages/react/src/components/RoomName.tsx +++ b/packages/react/src/components/RoomName.tsx @@ -19,10 +19,8 @@ export interface RoomNameProps extends React.HTMLAttributes { * * @param props - RoomNameProps */ -export const RoomName: ( - props: RoomNameProps & React.RefAttributes, -) => React.ReactNode = /* @__PURE__ */ React.forwardRef( - function RoomName( +export const RoomName: React.FC> = + /* @__PURE__ */ React.forwardRef(function RoomName( { childrenPosition = 'before', children, ...htmlAttributes }: RoomNameProps, ref, ) { @@ -35,5 +33,4 @@ export const RoomName: ( {childrenPosition === 'after' && children} ); - }, -); + }); diff --git a/packages/react/src/components/participant/ParticipantTile.tsx b/packages/react/src/components/participant/ParticipantTile.tsx index 9a5f3519b..5d0706e0b 100644 --- a/packages/react/src/components/participant/ParticipantTile.tsx +++ b/packages/react/src/components/participant/ParticipantTile.tsx @@ -51,8 +51,9 @@ export function ParticipantContextIfNeeded( /** * Only create a `TrackRefContext` if there is no `TrackRefContext` already. + * @internal */ -function TrackRefContextIfNeeded( +export function TrackRefContextIfNeeded( props: React.PropsWithChildren<{ trackRef?: TrackReferenceOrPlaceholder; }>, diff --git a/packages/react/src/context/index.ts b/packages/react/src/context/index.ts index b3e0ae0e1..5b5b49a26 100644 --- a/packages/react/src/context/index.ts +++ b/packages/react/src/context/index.ts @@ -24,3 +24,4 @@ export { } from './track-reference-context'; export { FeatureFlags, useFeatureContext, LKFeatureContext } from './feature-context'; +export { VoiceAssistantContext } from './voice-assistant-context'; diff --git a/packages/react/src/context/voice-assistant-context.ts b/packages/react/src/context/voice-assistant-context.ts new file mode 100644 index 000000000..7e266c85d --- /dev/null +++ b/packages/react/src/context/voice-assistant-context.ts @@ -0,0 +1,5 @@ +import * as React from 'react'; +import type { VoiceAssistant } from '../hooks/useVoiceAssistant'; + +/** @alpha */ +export const VoiceAssistantContext = React.createContext(undefined); diff --git a/packages/react/src/hooks/index.ts b/packages/react/src/hooks/index.ts index da8df0596..1c798c164 100644 --- a/packages/react/src/hooks/index.ts +++ b/packages/react/src/hooks/index.ts @@ -51,5 +51,6 @@ export { UseIsEncryptedOptions, useIsEncrypted } from './useIsEncrypted'; export * from './useTrackVolume'; export * from './useParticipantTracks'; export * from './useTrackTranscription'; +export * from './useVoiceAssistant'; export * from './useParticipantAttributes'; export * from './useIsRecording'; diff --git a/packages/react/src/hooks/useIsEncrypted.ts b/packages/react/src/hooks/useIsEncrypted.ts index 51e9b174c..198be9fec 100644 --- a/packages/react/src/hooks/useIsEncrypted.ts +++ b/packages/react/src/hooks/useIsEncrypted.ts @@ -17,12 +17,13 @@ export interface UseIsEncryptedOptions { */ export function useIsEncrypted(participant?: Participant, options: UseIsEncryptedOptions = {}) { const p = useEnsureParticipant(participant); + const room = useEnsureRoom(options.room); const observer = React.useMemo(() => encryptionStatusObservable(room, p), [room, p]); const isEncrypted = useObservableState( observer, - p instanceof LocalParticipant ? p.isE2EEEnabled : p.isEncrypted, + p instanceof LocalParticipant ? p.isE2EEEnabled : !!p?.isEncrypted, ); return isEncrypted; } diff --git a/packages/react/src/hooks/useParticipantAttributes.ts b/packages/react/src/hooks/useParticipantAttributes.ts index 1e6bc01fc..0e1d9bac0 100644 --- a/packages/react/src/hooks/useParticipantAttributes.ts +++ b/packages/react/src/hooks/useParticipantAttributes.ts @@ -1,7 +1,7 @@ import { participantAttributesObserver } from '@livekit/components-core'; import type { Participant } from 'livekit-client'; import * as React from 'react'; -import { useEnsureParticipant } from '../context'; +import { useEnsureParticipant, useMaybeParticipantContext } from '../context'; import { useObservableState } from './internal'; /** @@ -20,10 +20,15 @@ export interface UseParticipantAttributesOptions { /** @public */ export function useParticipantAttributes(props: UseParticipantAttributesOptions = {}) { - const p = useEnsureParticipant(props.participant); - const attributeObserver = React.useMemo(() => participantAttributesObserver(p), [p]); + const participantContext = useMaybeParticipantContext(); + const p = props.participant ?? participantContext; + const attributeObserver = React.useMemo( + // weird typescript constraint + () => (p ? participantAttributesObserver(p) : participantAttributesObserver(p)), + [p], + ); const { attributes } = useObservableState(attributeObserver, { - attributes: p.attributes, + attributes: p?.attributes, }); return { attributes }; @@ -47,6 +52,9 @@ export function useParticipantAttribute( const [attribute, setAttribute] = React.useState(p.attributes[attributeKey]); React.useEffect(() => { + if (!p) { + return; + } const subscription = participantAttributesObserver(p).subscribe((val) => { if (val.changed[attributeKey] !== undefined) { setAttribute(val.changed[attributeKey]); diff --git a/packages/react/src/hooks/useParticipantInfo.ts b/packages/react/src/hooks/useParticipantInfo.ts index 5d35379bd..d80c8bbac 100644 --- a/packages/react/src/hooks/useParticipantInfo.ts +++ b/packages/react/src/hooks/useParticipantInfo.ts @@ -1,7 +1,7 @@ import { participantInfoObserver } from '@livekit/components-core'; import type { Participant } from 'livekit-client'; import * as React from 'react'; -import { useEnsureParticipant } from '../context'; +import { useMaybeParticipantContext } from '../context'; import { useObservableState } from './internal'; /** @@ -20,12 +20,15 @@ export interface UseParticipantInfoOptions { /** @public */ export function useParticipantInfo(props: UseParticipantInfoOptions = {}) { - const p = useEnsureParticipant(props.participant); + let p = useMaybeParticipantContext(); + if (props.participant) { + p = props.participant; + } const infoObserver = React.useMemo(() => participantInfoObserver(p), [p]); const { identity, name, metadata } = useObservableState(infoObserver, { - name: p.name, - identity: p.identity, - metadata: p.metadata, + name: p?.name, + identity: p?.identity, + metadata: p?.metadata, }); return { identity, name, metadata }; diff --git a/packages/react/src/hooks/useTrackSyncTime.ts b/packages/react/src/hooks/useTrackSyncTime.ts index a1f2ef737..0a5b54fec 100644 --- a/packages/react/src/hooks/useTrackSyncTime.ts +++ b/packages/react/src/hooks/useTrackSyncTime.ts @@ -5,13 +5,13 @@ import { useObservableState } from './internal'; /** * @internal */ -export function useTrackSyncTime({ publication }: TrackReferenceOrPlaceholder) { +export function useTrackSyncTime(ref: TrackReferenceOrPlaceholder | undefined) { const observable = React.useMemo( - () => (publication?.track ? trackSyncTimeObserver(publication.track) : undefined), - [publication?.track], + () => (ref?.publication?.track ? trackSyncTimeObserver(ref?.publication.track) : undefined), + [ref?.publication?.track], ); return useObservableState(observable, { timestamp: Date.now(), - rtpTimestamp: publication?.track?.rtpTimestamp, + rtpTimestamp: ref?.publication?.track?.rtpTimestamp, }); } diff --git a/packages/react/src/hooks/useTrackTranscription.ts b/packages/react/src/hooks/useTrackTranscription.ts index 3faa7634c..809dd37b4 100644 --- a/packages/react/src/hooks/useTrackTranscription.ts +++ b/packages/react/src/hooks/useTrackTranscription.ts @@ -39,7 +39,7 @@ const TRACK_TRANSCRIPTION_DEFAULTS = { * @alpha */ export function useTrackTranscription( - trackRef: TrackReferenceOrPlaceholder, + trackRef: TrackReferenceOrPlaceholder | undefined, options?: TrackTranscriptionOptions, ) { const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options }; @@ -61,7 +61,7 @@ export function useTrackTranscription( ); }; React.useEffect(() => { - if (!trackRef.publication) { + if (!trackRef?.publication) { return; } const subscription = trackTranscriptionObserver(trackRef.publication).subscribe((evt) => { @@ -70,7 +70,7 @@ export function useTrackTranscription( return () => { subscription.unsubscribe(); }; - }, [getTrackReferenceId(trackRef), handleSegmentMessage]); + }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]); // React.useEffect(() => { // if (syncTimestamps) { diff --git a/packages/react/src/hooks/useVoiceAssistant.ts b/packages/react/src/hooks/useVoiceAssistant.ts new file mode 100644 index 000000000..152fa86db --- /dev/null +++ b/packages/react/src/hooks/useVoiceAssistant.ts @@ -0,0 +1,63 @@ +import * as React from 'react'; +import { ConnectionState, ParticipantKind, Track } from 'livekit-client'; +import type { RemoteParticipant } from 'livekit-client'; +import type { ReceivedTranscriptionSegment, TrackReference } from '@livekit/components-core'; +import { useRemoteParticipants } from './useRemoteParticipants'; +import { useParticipantTracks } from './useParticipantTracks'; +import { useTrackTranscription } from './useTrackTranscription'; +import { useConnectionState } from './useConnectionStatus'; +import { useParticipantAttributes } from './useParticipantAttributes'; + +/** + * @alpha + */ +export type VoiceAssistantState = + | 'disconnected' + | 'connecting' + | 'listening' + | 'thinking' + | 'speaking'; + +/** + * @alpha + */ +export interface VoiceAssistant { + agent: RemoteParticipant | undefined; + state: VoiceAssistantState; + audioTrack: TrackReference | undefined; + agentTranscriptions: ReceivedTranscriptionSegment[]; + agentAttributes: RemoteParticipant['attributes'] | undefined; +} + +/** + * @alpha + * + * This hook looks for the first agent-participant in the room. + * It assumes that the agent participant is based on the LiveKit VoiceAssistant API and + * returns the most commonly used state vars when interacting with a VoiceAssistant. + */ +export function useVoiceAssistant(): VoiceAssistant { + const agent = useRemoteParticipants().find((p) => p.kind === ParticipantKind.AGENT); + const audioTrack = useParticipantTracks([Track.Source.Microphone], agent?.identity)[0]; + const { segments: agentTranscriptions } = useTrackTranscription(audioTrack); + const connectionState = useConnectionState(); + const { attributes } = useParticipantAttributes({ participant: agent }); + + const state: VoiceAssistantState = React.useMemo(() => { + if (connectionState === ConnectionState.Disconnected) { + return 'disconnected'; + } else if (connectionState === ConnectionState.Connecting || !agent || !attributes?.state) { + return 'connecting'; + } else { + return attributes.state as VoiceAssistantState; + } + }, [attributes, agent, connectionState]); + + return { + agent, + state, + audioTrack, + agentTranscriptions, + agentAttributes: attributes, + }; +}