Add useVoiceAssistant (#917)

.changeset/dull-pots-applaud.md

@@ -0,0 +1,7 @@
+---
+"@livekit/components-core": patch
+"@livekit/components-react": patch
+"eslint-config-lk-custom": patch
+---
+
+Add useVoiceAssistant

packages/core/etc/components-core.api.md

@@ -15,7 +15,7 @@ import type { LocalParticipant } from 'livekit-client';
 import { LocalVideoTrack } from 'livekit-client';
 import loglevel from 'loglevel';
 import { Observable } from 'rxjs';
-import type { Participant } from 'livekit-client';
+import { Participant } from 'livekit-client';
 import { ParticipantEvent } from 'livekit-client';
 import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant';
 import type { ParticipantKind } from 'livekit-client';
@@ -174,7 +174,7 @@ export const defaultUserChoices: LocalUserChoices;
 export function didActiveSegmentsChange<T extends TranscriptionSegment>(prevActive: T[], newActive: T[]): boolean;
 
 // @public (undocumented)
-export function encryptionStatusObservable(room: Room, participant: Participant): Observable<boolean>;
+export function encryptionStatusObservable(room: Room, participant: Participant | undefined): Observable<boolean>;
 
 // @public (undocumented)
 export function getActiveTranscriptionSegments(segments: ReceivedTranscriptionSegment[], syncTimes: {
@@ -312,9 +312,12 @@ export function observeTrackEvents(track: TrackPublication, ...events: TrackEven
 export function participantAttributesObserver(participant: Participant): Observable<{
     changed: Readonly<Record<string, string>>;
     attributes: Readonly<Record<string, string>>;
-} | {
-    changed: Readonly<Record<string, string>>;
-    attributes: Readonly<Record<string, string>>;
+}>;
+
+// @public (undocumented)
+export function participantAttributesObserver(participant: undefined): Observable<{
+    changed: undefined;
+    attributes: undefined;
 }>;
 
 // Warning: (ae-incompatible-release-tags) The symbol "participantByIdentifierObserver" is marked as @public, but its signature references "ParticipantIdentifier" which is marked as @beta
@@ -347,15 +350,15 @@ export type ParticipantIdentifier = RequireAtLeastOne<{
 }, 'identity' | 'kind'>;
 
 // @public (undocumented)
-export function participantInfoObserver(participant: Participant): Observable<{
+export function participantInfoObserver(participant?: Participant): Observable<{
     name: string | undefined;
     identity: string;
     metadata: string | undefined;
 } | {
     name: string | undefined;
     identity: string;
     metadata: string | undefined;
-}>;
+}> | undefined;
 
 // @public (undocumented)
 export interface ParticipantMedia<T extends Participant = Participant> {
@@ -572,7 +575,7 @@ export function setupParticipantName(participant: Participant): {
     name: string | undefined;
     identity: string;
     metadata: string | undefined;
-    }>;
+    }> | undefined;
 };
 
 // @public (undocumented)

packages/core/src/observables/participant.ts

@@ -1,5 +1,5 @@
 import type { ParticipantPermission } from '@livekit/protocol';
-import type { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client';
+import { Participant, RemoteParticipant, Room, TrackPublication } from 'livekit-client';
 import { ParticipantEvent, RoomEvent, Track } from 'livekit-client';
 import type { ParticipantEventCallbacks } from 'livekit-client/dist/src/room/participant/Participant';
 import type { Subscriber } from 'rxjs';
@@ -85,7 +85,10 @@ export function createTrackObserver(participant: Participant, options: TrackIden
   );
 }
 
-export function participantInfoObserver(participant: Participant) {
+export function participantInfoObserver(participant?: Participant) {
+  if (!participant) {
+    return undefined;
+  }
   const observer = observeParticipantEvents(
     participant,
     ParticipantEvent.ParticipantMetadataChanged,
@@ -287,7 +290,18 @@ export function participantByIdentifierObserver(
   return observable;
 }
 
-export function participantAttributesObserver(participant: Participant) {
+export function participantAttributesObserver(participant: Participant): Observable<{
+  changed: Readonly<Record<string, string>>;
+  attributes: Readonly<Record<string, string>>;
+}>;
+export function participantAttributesObserver(participant: undefined): Observable<{
+  changed: undefined;
+  attributes: undefined;
+}>;
+export function participantAttributesObserver(participant: Participant | undefined) {
+  if (typeof participant === 'undefined') {
+    return new Observable<{ changed: undefined; attributes: undefined }>();
+  }
   return participantEventSelector(participant, ParticipantEvent.AttributesChanged).pipe(
     map(([changedAttributes]) => {
       return {

packages/core/src/observables/room.ts

@@ -248,16 +248,18 @@ export function createActiveDeviceObservable(room: Room, kind: MediaDeviceKind)
   );
 }
 
-export function encryptionStatusObservable(room: Room, participant: Participant) {
+export function encryptionStatusObservable(room: Room, participant: Participant | undefined) {
   return roomEventSelector(room, RoomEvent.ParticipantEncryptionStatusChanged).pipe(
     filter(
       ([, p]) =>
-        participant.identity === p?.identity ||
-        (!p && participant.identity === room.localParticipant.identity),
+        participant?.identity === p?.identity ||
+        (!p && participant?.identity === room.localParticipant.identity),
     ),
     map(([encrypted]) => encrypted),
     startWith(
-      participant instanceof LocalParticipant ? participant.isE2EEEnabled : participant.isEncrypted,
+      participant instanceof LocalParticipant
+        ? participant.isE2EEEnabled
+        : !!participant?.isEncrypted,
     ),
   );
 }

packages/react/etc/components-react.api.md

@@ -573,7 +573,7 @@ export interface RoomAudioRendererProps {
 export const RoomContext: React_2.Context<Room | undefined>;
 
 // @public
-export const RoomName: (props: RoomNameProps & React_2.RefAttributes<HTMLSpanElement>) => React_2.ReactNode;
+export const RoomName: React_2.FC<RoomNameProps & React_2.RefAttributes<HTMLSpanElement>>;
 
 // @public (undocumented)
 export interface RoomNameProps extends React_2.HTMLAttributes<HTMLSpanElement> {
@@ -637,6 +637,13 @@ export interface TrackMutedIndicatorProps extends React_2.HTMLAttributes<HTMLDiv
 // @public
 export const TrackRefContext: React_2.Context<TrackReferenceOrPlaceholder | undefined>;
 
+// Warning: (ae-internal-missing-underscore) The name "TrackRefContextIfNeeded" should be prefixed with an underscore because the declaration is marked as @internal
+//
+// @internal
+export function TrackRefContextIfNeeded(props: React_2.PropsWithChildren<{
+    trackRef?: TrackReferenceOrPlaceholder;
+}>): React_2.JSX.Element;
+
 // @public (undocumented)
 export type TrackReference = {
     participant: Participant;
@@ -881,7 +888,7 @@ export function useMediaDeviceSelect({ kind, room, track, requestPermissions, on
     devices: MediaDeviceInfo[];
     className: string;
     activeDeviceId: string;
-    setActiveMediaDevice: (id: string, options?: SetMediaDeviceOptions) => Promise<void>;
+    setActiveMediaDevice: (id: string, options?: SetMediaDeviceOptions | undefined) => Promise<void>;
 };
 
 // @public (undocumented)
@@ -916,7 +923,7 @@ export function useParticipantAttribute(attributeKey: string, options?: UseParti
 
 // @public (undocumented)
 export function useParticipantAttributes(props?: UseParticipantAttributesOptions): {
-    attributes: Readonly<Record<string, string>>;
+    attributes: Readonly<Record<string, string>> | undefined;
 };
 
 // @public
@@ -930,7 +937,7 @@ export function useParticipantContext(): Participant;
 
 // @public (undocumented)
 export function useParticipantInfo(props?: UseParticipantInfoOptions): {
-    identity: string;
+    identity: string | undefined;
     name: string | undefined;
     metadata: string | undefined;
 };
@@ -1152,7 +1159,7 @@ export type UseTracksOptions = {
 
 // @public
 export function useTrackToggle<T extends ToggleSource>({ source, onChange, initialState, captureOptions, publishOptions, onDeviceError, ...rest }: UseTrackToggleProps<T>): {
-    toggle: (forceState?: boolean, captureOptions?: CaptureOptionsBySource<T> | undefined) => Promise<void>;
+    toggle: (forceState?: boolean | undefined, captureOptions?: CaptureOptionsBySource<T> | undefined) => Promise<void>;
     enabled: boolean;
     pending: boolean;
     track: LocalTrackPublication | undefined;
@@ -1164,7 +1171,7 @@ export interface UseTrackToggleProps<T extends ToggleSource> extends Omit<TrackT
 }
 
 // @alpha (undocumented)
-export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder, options?: TrackTranscriptionOptions): {
+export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder | undefined, options?: TrackTranscriptionOptions): {
     segments: ReceivedTranscriptionSegment[];
 };
 
@@ -1180,6 +1187,9 @@ export interface UseVisualStableUpdateOptions {
     customSortFunction?: (trackReferences: TrackReferenceOrPlaceholder[]) => TrackReferenceOrPlaceholder[];
 }
 
+// @alpha
+export function useVoiceAssistant(): VoiceAssistant;
+
 // @public
 export function VideoConference({ chatMessageFormatter, chatMessageDecoder, chatMessageEncoder, SettingsComponent, ...props }: VideoConferenceProps): React_2.JSX.Element;
 
@@ -1211,6 +1221,26 @@ export interface VideoTrackProps extends React_2.VideoHTMLAttributes<HTMLVideoEl
     trackRef?: TrackReference;
 }
 
+// @alpha (undocumented)
+export interface VoiceAssistant {
+    // (undocumented)
+    agent: RemoteParticipant | undefined;
+    // (undocumented)
+    agentAttributes: RemoteParticipant['attributes'] | undefined;
+    // (undocumented)
+    agentTranscriptions: ReceivedTranscriptionSegment[];
+    // (undocumented)
+    audioTrack: TrackReference | undefined;
+    // (undocumented)
+    state: VoiceAssistantState;
+}
+
+// @alpha (undocumented)
+export const VoiceAssistantContext: React_2.Context<VoiceAssistant | undefined>;
+
+// @alpha (undocumented)
+export type VoiceAssistantState = 'disconnected' | 'connecting' | 'listening' | 'thinking' | 'speaking';
+
 // @public (undocumented)
 export type WidgetState = {
     showChat: boolean;

packages/react/src/components/RoomName.tsx

@@ -19,10 +19,8 @@ export interface RoomNameProps extends React.HTMLAttributes<HTMLSpanElement> {
  *
  * @param props - RoomNameProps
  */
-export const RoomName: (
-  props: RoomNameProps & React.RefAttributes<HTMLSpanElement>,
-) => React.ReactNode = /* @__PURE__ */ React.forwardRef<HTMLSpanElement, RoomNameProps>(
-  function RoomName(
+export const RoomName: React.FC<RoomNameProps & React.RefAttributes<HTMLSpanElement>> =
+  /* @__PURE__ */ React.forwardRef<HTMLSpanElement, RoomNameProps>(function RoomName(
     { childrenPosition = 'before', children, ...htmlAttributes }: RoomNameProps,
     ref,
   ) {
@@ -35,5 +33,4 @@ export const RoomName: (
         {childrenPosition === 'after' && children}
       </span>
     );
-  },
-);
+  });

packages/react/src/components/participant/ParticipantTile.tsx

@@ -51,8 +51,9 @@ export function ParticipantContextIfNeeded(
 
 /**
  * Only create a `TrackRefContext` if there is no `TrackRefContext` already.
+ * @internal
  */
-function TrackRefContextIfNeeded(
+export function TrackRefContextIfNeeded(
   props: React.PropsWithChildren<{
     trackRef?: TrackReferenceOrPlaceholder;
   }>,

packages/react/src/context/index.ts

@@ -24,3 +24,4 @@ export {
 } from './track-reference-context';
 
 export { FeatureFlags, useFeatureContext, LKFeatureContext } from './feature-context';
+export { VoiceAssistantContext } from './voice-assistant-context';

packages/react/src/context/voice-assistant-context.ts

@@ -0,0 +1,5 @@
+import * as React from 'react';
+import type { VoiceAssistant } from '../hooks/useVoiceAssistant';
+
+/** @alpha */
+export const VoiceAssistantContext = React.createContext<VoiceAssistant | undefined>(undefined);

packages/react/src/hooks/index.ts

@@ -51,5 +51,6 @@ export { UseIsEncryptedOptions, useIsEncrypted } from './useIsEncrypted';
 export * from './useTrackVolume';
 export * from './useParticipantTracks';
 export * from './useTrackTranscription';
+export * from './useVoiceAssistant';
 export * from './useParticipantAttributes';
 export * from './useIsRecording';

packages/react/src/hooks/useIsEncrypted.ts

@@ -17,12 +17,13 @@ export interface UseIsEncryptedOptions {
  */
 export function useIsEncrypted(participant?: Participant, options: UseIsEncryptedOptions = {}) {
   const p = useEnsureParticipant(participant);
+
   const room = useEnsureRoom(options.room);
 
   const observer = React.useMemo(() => encryptionStatusObservable(room, p), [room, p]);
   const isEncrypted = useObservableState(
     observer,
-    p instanceof LocalParticipant ? p.isE2EEEnabled : p.isEncrypted,
+    p instanceof LocalParticipant ? p.isE2EEEnabled : !!p?.isEncrypted,
   );
   return isEncrypted;
 }

packages/react/src/hooks/useParticipantAttributes.ts

@@ -1,7 +1,7 @@
 import { participantAttributesObserver } from '@livekit/components-core';
 import type { Participant } from 'livekit-client';
 import * as React from 'react';
-import { useEnsureParticipant } from '../context';
+import { useEnsureParticipant, useMaybeParticipantContext } from '../context';
 import { useObservableState } from './internal';
 
 /**
@@ -20,10 +20,15 @@ export interface UseParticipantAttributesOptions {
 
 /** @public */
 export function useParticipantAttributes(props: UseParticipantAttributesOptions = {}) {
-  const p = useEnsureParticipant(props.participant);
-  const attributeObserver = React.useMemo(() => participantAttributesObserver(p), [p]);
+  const participantContext = useMaybeParticipantContext();
+  const p = props.participant ?? participantContext;
+  const attributeObserver = React.useMemo(
+    // weird typescript constraint
+    () => (p ? participantAttributesObserver(p) : participantAttributesObserver(p)),
+    [p],
+  );
   const { attributes } = useObservableState(attributeObserver, {
-    attributes: p.attributes,
+    attributes: p?.attributes,
   });
 
   return { attributes };
@@ -47,6 +52,9 @@ export function useParticipantAttribute(
   const [attribute, setAttribute] = React.useState(p.attributes[attributeKey]);
 
   React.useEffect(() => {
+    if (!p) {
+      return;
+    }
     const subscription = participantAttributesObserver(p).subscribe((val) => {
       if (val.changed[attributeKey] !== undefined) {
         setAttribute(val.changed[attributeKey]);

packages/react/src/hooks/useParticipantInfo.ts

@@ -1,7 +1,7 @@
 import { participantInfoObserver } from '@livekit/components-core';
 import type { Participant } from 'livekit-client';
 import * as React from 'react';
-import { useEnsureParticipant } from '../context';
+import { useMaybeParticipantContext } from '../context';
 import { useObservableState } from './internal';
 
 /**
@@ -20,12 +20,15 @@ export interface UseParticipantInfoOptions {
 
 /** @public */
 export function useParticipantInfo(props: UseParticipantInfoOptions = {}) {
-  const p = useEnsureParticipant(props.participant);
+  let p = useMaybeParticipantContext();
+  if (props.participant) {
+    p = props.participant;
+  }
   const infoObserver = React.useMemo(() => participantInfoObserver(p), [p]);
   const { identity, name, metadata } = useObservableState(infoObserver, {
-    name: p.name,
-    identity: p.identity,
-    metadata: p.metadata,
+    name: p?.name,
+    identity: p?.identity,
+    metadata: p?.metadata,
   });
 
   return { identity, name, metadata };

packages/react/src/hooks/useTrackSyncTime.ts

@@ -5,13 +5,13 @@ import { useObservableState } from './internal';
 /**
  * @internal
  */
-export function useTrackSyncTime({ publication }: TrackReferenceOrPlaceholder) {
+export function useTrackSyncTime(ref: TrackReferenceOrPlaceholder | undefined) {
   const observable = React.useMemo(
-    () => (publication?.track ? trackSyncTimeObserver(publication.track) : undefined),
-    [publication?.track],
+    () => (ref?.publication?.track ? trackSyncTimeObserver(ref?.publication.track) : undefined),
+    [ref?.publication?.track],
   );
   return useObservableState(observable, {
     timestamp: Date.now(),
-    rtpTimestamp: publication?.track?.rtpTimestamp,
+    rtpTimestamp: ref?.publication?.track?.rtpTimestamp,
   });
 }