Add activeSegements to useTrackTranscription

.changeset/real-crabs-beam.md

@@ -0,0 +1,6 @@
+---
+"@livekit/components-core": patch
+"@livekit/components-react": patch
+---
+
+Add activeSegements to useTrackTranscription

.github/workflows/release.yaml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v2
+      - uses: pnpm/action-setup@v4
       - name: Use Node.js 20
         uses: actions/setup-node@v4
         with:

.github/workflows/size-limit.yaml

@@ -11,7 +11,7 @@ jobs:
       CI_JOB_NUMBER: 1
     steps:
       - uses: actions/checkout@v4
-      - uses: pnpm/action-setup@v2
+      - uses: pnpm/action-setup@v4
       - name: Use Node.js 20
         uses: actions/setup-node@v4
         with:

.github/workflows/test.yaml

@@ -16,7 +16,7 @@ jobs:
         with:
           fetch-depth: 2
 
-      - uses: pnpm/action-setup@v2
+      - uses: pnpm/action-setup@v4
 
       - name: Use Node.js 20
         uses: actions/setup-node@v4

examples/nextjs/pages/transcriptions.tsx

@@ -0,0 +1,77 @@
+import {
+  LiveKitRoom,
+  useToken,
+  setLogLevel,
+  useParticipantTracks,
+  useTrackTranscription,
+} from '@livekit/components-react';
+import { Track } from 'livekit-client';
+import type { NextPage } from 'next';
+import { generateRandomUserId } from '../lib/helper';
+import { useMemo } from 'react';
+
+const TranscriptionExample: NextPage = () => {
+  const params = typeof window !== 'undefined' ? new URLSearchParams(location.search) : null;
+  const roomName = useMemo(() => params?.get('room') ?? generateRandomUserId(), []);
+  setLogLevel('info', { liveKitClientLogLevel: 'debug' });
+  const userId = useMemo(() => params?.get('user') ?? 'test-user', []);
+
+  const tokenOptions = useMemo(() => {
+    return {
+      userInfo: {
+        identity: userId,
+        name: userId,
+      },
+    };
+  }, [userId]);
+
+  const token = useToken(process.env.NEXT_PUBLIC_LK_TOKEN_ENDPOINT, roomName, tokenOptions);
+
+  return (
+    <div data-lk-theme="default" style={{ height: '100vh' }}>
+      <LiveKitRoom
+        video={false}
+        audio={true}
+        token={token}
+        serverUrl={process.env.NEXT_PUBLIC_LK_SERVER_URL}
+        onMediaDeviceFailure={(e) => {
+          console.error(e);
+          alert(
+            'Error acquiring camera or microphone permissions. Please make sure you grant the necessary permissions in your browser and reload the tab',
+          );
+        }}
+      >
+        <div style={{ display: 'grid', gridTemplateColumns: '50% 50%' }}>
+          <div>
+            <h2>All</h2>
+            <ParticipantTranscriptions identity={userId} activeOnly={false} />
+          </div>
+          <div>
+            <h2>Active only</h2>
+            <ParticipantTranscriptions identity={userId} activeOnly={true} />
+          </div>
+        </div>
+      </LiveKitRoom>
+    </div>
+  );
+};
+
+function ParticipantTranscriptions({
+  identity,
+  activeOnly = true,
+  ...props
+}: React.HtmlHTMLAttributes<HTMLDivElement> & { identity: string; activeOnly?: boolean }) {
+  const audioTracks = useParticipantTracks([Track.Source.Microphone], identity);
+  const transcriptions = useTrackTranscription(audioTracks[0]);
+  const segments = activeOnly ? transcriptions.activeSegments : transcriptions.segments;
+
+  return (
+    <div {...props}>
+      {segments.map((segment) => (
+        <p key={segment.id}>{segment.text}</p>
+      ))}
+    </div>
+  );
+}
+
+export default TranscriptionExample;

packages/core/src/helper/transcriptions.ts

@@ -8,11 +8,11 @@ export type ReceivedTranscriptionSegment = TranscriptionSegment & {
 export function getActiveTranscriptionSegments(
   segments: ReceivedTranscriptionSegment[],
   syncTimes: { timestamp: number; rtpTimestamp?: number },
-  maxAge = 0,
+  maxAge = 5_000,
 ) {
   return segments.filter((segment) => {
-    const hasTrackSync = !!syncTimes.rtpTimestamp;
-    const currentTrackTime = syncTimes.rtpTimestamp ?? performance.timeOrigin + performance.now();
+    const hasTrackSync = !!syncTimes.rtpTimestamp && segment.startTime !== 0;
+    const currentTrackTime = syncTimes.rtpTimestamp || performance.timeOrigin + performance.now();
     // if a segment arrives late, consider startTime to be the media timestamp from when the segment was received client side
     const displayStartTime = hasTrackSync
       ? Math.max(segment.receivedAtMediaTimestamp, segment.startTime)
@@ -29,6 +29,7 @@ export function addMediaTimestampToTranscription(
   segment: TranscriptionSegment,
   timestamps: { timestamp: number; rtpTimestamp?: number },
 ): ReceivedTranscriptionSegment {
+  console.log('new media timestamp', { segment, receivedAt: timestamps.timestamp });
   return {
     ...segment,
     receivedAtMediaTimestamp: timestamps.rtpTimestamp ?? 0,

packages/react/etc/components-react.api.md

@@ -667,6 +667,7 @@ export interface TrackToggleProps<T extends ToggleSource> extends Omit<React_2.B
 // @alpha (undocumented)
 export interface TrackTranscriptionOptions {
     bufferSize?: number;
+    maxAge?: number;
 }
 
 // Warning: (ae-internal-missing-underscore) The name "UnfocusToggleIcon" should be prefixed with an underscore because the declaration is marked as @internal
@@ -1133,8 +1134,9 @@ export interface UseTrackToggleProps<T extends ToggleSource> extends Omit<TrackT
 }
 
 // @alpha (undocumented)
-export function useTrackTranscription(trackRef: TrackReferenceOrPlaceholder, options?: TrackTranscriptionOptions): {
+export function useTrackTranscription(trackRef?: TrackReferenceOrPlaceholder, options?: TrackTranscriptionOptions): {
     segments: ReceivedTranscriptionSegment[];
+    activeSegments: ReceivedTranscriptionSegment[];
 };
 
 // @alpha

packages/react/src/hooks/useParticipantTracks.ts

@@ -4,6 +4,7 @@ import { participantTracksObservable } from '@livekit/components-core';
 import { useObservableState } from './internal';
 import type { Track } from 'livekit-client';
 import { useMaybeParticipantContext, useRoomContext } from '../context';
+import { useConnectionState } from './useConnectionStatus';
 
 /**
  * `useParticipantTracks` is a custom React that allows you to get tracks of a specific participant only, by specifiying the participant's identity.
@@ -16,9 +17,16 @@ export function useParticipantTracks(
 ): TrackReference[] {
   const room = useRoomContext();
   const participantContext = useMaybeParticipantContext();
-  const p = participantIdentity
-    ? room.getParticipantByIdentity(participantIdentity)
-    : participantContext;
+  const connectionState = useConnectionState();
+  const p = React.useMemo(
+    () =>
+      participantIdentity
+        ? participantIdentity === room.localParticipant.identity
+          ? room.localParticipant
+          : room.getParticipantByIdentity(participantIdentity)
+        : participantContext,
+    [connectionState, participantIdentity, room, participantContext],
+  );
   const observable = React.useMemo(
     () => (p ? participantTracksObservable(p, { sources }) : undefined),
     [p?.sid, p?.identity, JSON.stringify(sources)],

packages/react/src/hooks/useTrackSyncTime.ts

@@ -5,13 +5,14 @@ import { useObservableState } from './internal';
 /**
  * @internal
  */
-export function useTrackSyncTime({ publication }: TrackReferenceOrPlaceholder) {
+export function useTrackSyncTime(trackRef?: TrackReferenceOrPlaceholder) {
   const observable = React.useMemo(
-    () => (publication?.track ? trackSyncTimeObserver(publication.track) : undefined),
-    [publication?.track],
+    () =>
+      trackRef?.publication?.track ? trackSyncTimeObserver(trackRef?.publication.track) : undefined,
+    [trackRef?.publication?.track],
   );
   return useObservableState(observable, {
-    timestamp: Date.now(),
-    rtpTimestamp: publication?.track?.rtpTimestamp,
+    timestamp: performance.timeOrigin + performance.now(),
+    rtpTimestamp: trackRef?.publication?.track?.rtpTimestamp,
   });
 }

packages/react/src/hooks/useTrackTranscription.ts

@@ -1,12 +1,12 @@
 import {
   type ReceivedTranscriptionSegment,
-  addMediaTimestampToTranscription as addTimestampsToTranscription,
+  addMediaTimestampToTranscription,
   dedupeSegments,
-  // getActiveTranscriptionSegments,
+  getActiveTranscriptionSegments,
   getTrackReferenceId,
   trackTranscriptionObserver,
   type TrackReferenceOrPlaceholder,
-  // didActiveSegmentsChange,
+  didActiveSegmentsChange,
 } from '@livekit/components-core';
 import type { TranscriptionSegment } from 'livekit-client';
 import * as React from 'react';
@@ -22,41 +22,44 @@ export interface TrackTranscriptionOptions {
    */
   bufferSize?: number;
   /** amount of time (in ms) that the segment is considered `active` past its original segment duration, defaults to 2_000 */
-  // maxAge?: number;
+  maxAge?: number;
 }
 
 const TRACK_TRANSCRIPTION_DEFAULTS = {
   bufferSize: 100,
-  // maxAge: 2_000,
+  maxAge: 5_000,
 } as const satisfies TrackTranscriptionOptions;
 
 /**
  * @returns An object consisting of `segments` with maximum length of opts.windowLength and `activeSegments` that are valid for the current track timestamp
  * @alpha
  */
 export function useTrackTranscription(
-  trackRef: TrackReferenceOrPlaceholder,
+  trackRef?: TrackReferenceOrPlaceholder,
   options?: TrackTranscriptionOptions,
 ) {
   const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options };
   const [segments, setSegments] = React.useState<Array<ReceivedTranscriptionSegment>>([]);
-  // const [activeSegments, setActiveSegments] = React.useState<Array<ReceivedTranscriptionSegment>>(
-  //   [],
-  // );
-  // const prevActiveSegments = React.useRef<ReceivedTranscriptionSegment[]>([]);
+  const [activeSegments, setActiveSegments] = React.useState<Array<ReceivedTranscriptionSegment>>(
+    [],
+  );
+  const prevActiveSegments = React.useRef<ReceivedTranscriptionSegment[]>([]);
   const syncTimestamps = useTrackSyncTime(trackRef);
-  const handleSegmentMessage = (newSegments: TranscriptionSegment[]) => {
-    setSegments((prevSegments) =>
-      dedupeSegments(
-        prevSegments,
-        // when first receiving a segment, add the current media timestamp to it
-        newSegments.map((s) => addTimestampsToTranscription(s, syncTimestamps)),
-        opts.bufferSize,
-      ),
-    );
-  };
+  const handleSegmentMessage = React.useCallback(
+    (newSegments: TranscriptionSegment[]) => {
+      setSegments((prevSegments) =>
+        dedupeSegments(
+          prevSegments,
+          // when first receiving a segment, add the current media timestamp to it
+          newSegments.map((s) => addMediaTimestampToTranscription(s, syncTimestamps)),
+          opts.bufferSize,
+        ),
+      );
+    },
+    [syncTimestamps, opts.bufferSize],
+  );
   React.useEffect(() => {
-    if (!trackRef.publication) {
+    if (!trackRef?.publication) {
       return;
     }
     const subscription = trackTranscriptionObserver(trackRef.publication).subscribe((evt) => {
@@ -65,22 +68,21 @@ export function useTrackTranscription(
     return () => {
       subscription.unsubscribe();
     };
-  }, [getTrackReferenceId(trackRef), handleSegmentMessage]);
+  }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]);
 
-  // React.useEffect(() => {
-  //   if (syncTimestamps) {
-  //     const newActiveSegments = getActiveTranscriptionSegments(
-  //       segments,
-  //       syncTimestamps,
-  //       opts.maxAge,
-  //     );
-  //     // only update active segment array if content actually changed
-  //     if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) {
-  //       setActiveSegments(newActiveSegments);
-  //       prevActiveSegments.current = newActiveSegments;
-  //     }
-  //   }
-  // }, [syncTimestamps, segments, opts.maxAge]);
-
-  return { segments };
+  React.useEffect(() => {
+    if (syncTimestamps) {
+      const newActiveSegments = getActiveTranscriptionSegments(
+        segments,
+        syncTimestamps,
+        opts.maxAge,
+      );
+      // only update active segment array if content actually changed
+      if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) {
+        setActiveSegments(newActiveSegments);
+        prevActiveSegments.current = newActiveSegments;
+      }
+    }
+  }, [syncTimestamps, segments, opts.maxAge]);
+  return { segments, activeSegments };
 }