Skip to content

Commit

Permalink
Add timestamps to TranscriptionSegment (#459)
Browse files Browse the repository at this point in the history
This adds tracking of transcription receipt times which simplifies the
consumption of this feature. I also removed startTime and endTime from
the version we expose, since neither is used yet at the protocol level.

The new fields are `firstReceivedTime` and `lastReceivedTime`.

I tried this out in my sample project and it really simplified things.
Now I can implement a very basic real-time chat thread without needing
multiple data structures or new types. The only thing that still
requires extra bookkeeping is tracking participant name, etc, since
TranscriptionSegment lives under TrackPublication and Participant, and
we don't denormalize the Participant id down the chain.

See the [associated commit in my docs
PR](livekit/web@0076c75)
for a visual of how this simplifies basic uses
  • Loading branch information
bcherry authored Aug 12, 2024
1 parent 99cad65 commit b55889c
Show file tree
Hide file tree
Showing 3 changed files with 29 additions and 22 deletions.
23 changes: 20 additions & 3 deletions Sources/LiveKit/Core/Room+EngineDelegate.swift
Original file line number Diff line number Diff line change
Expand Up @@ -207,13 +207,30 @@ extension Room {
return
}

let segments = packet.segments.map { $0.toLKType() }

guard !segments.isEmpty else {
guard !packet.segments.isEmpty else {
log("[Transcription] Received segments are empty", .warning)
return
}

let segments = packet.segments.map { segment in
TranscriptionSegment(id: segment.id,
text: segment.text,
language: segment.language,
firstReceivedTime: _state.transcriptionReceivedTimes[segment.id] ?? Date(),
lastReceivedTime: Date(),
isFinal: segment.final)
}

_state.mutate { state in
for segment in segments {
if segment.isFinal {
state.transcriptionReceivedTimes.removeValue(forKey: segment.id)
} else {
state.transcriptionReceivedTimes[segment.id] = segment.firstReceivedTime
}
}
}

delegates.notify {
$0.room?(self, participant: participant, trackPublication: publication, didReceiveTranscriptionSegments: segments)
}
Expand Down
3 changes: 3 additions & 0 deletions Sources/LiveKit/Core/Room.swift
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ public class Room: NSObject, ObservableObject, Loggable {
var publisher: Transport?
var subscriber: Transport?
var isSubscriberPrimary: Bool = false

// Agents
var transcriptionReceivedTimes: [String: Date] = [:]

@discardableResult
mutating func updateRemoteParticipant(info: Livekit_ParticipantInfo, room: Room) -> RemoteParticipant {
Expand Down
25 changes: 6 additions & 19 deletions Sources/LiveKit/Types/TranscriptionSegment.swift
Original file line number Diff line number Diff line change
Expand Up @@ -21,22 +21,22 @@ public class TranscriptionSegment: NSObject {
public let id: String
public let text: String
public let language: String
public let startTime: UInt64
public let endTime: UInt64
public let firstReceivedTime: Date
public let lastReceivedTime: Date
public let isFinal: Bool

init(id: String,
text: String,
language: String,
startTime: UInt64,
endTime: UInt64,
firstReceivedTime: Date,
lastReceivedTime: Date,
isFinal: Bool)
{
self.id = id
self.text = text
self.language = language
self.startTime = startTime
self.endTime = endTime
self.firstReceivedTime = firstReceivedTime
self.lastReceivedTime = lastReceivedTime
self.isFinal = isFinal
}

Expand All @@ -53,16 +53,3 @@ public class TranscriptionSegment: NSObject {
return hasher.finalize()
}
}

// MARK: - Internal

extension Livekit_TranscriptionSegment {
func toLKType() -> TranscriptionSegment {
TranscriptionSegment(id: id,
text: text,
language: language,
startTime: startTime,
endTime: endTime,
isFinal: final)
}
}

0 comments on commit b55889c

Please sign in to comment.