From eaf1e24aaa31b24cb207d37992a9075c3ed0ca49 Mon Sep 17 00:00:00 2001 From: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Date: Thu, 8 Aug 2024 23:07:10 +0800 Subject: [PATCH] Transcription events CLT-343 (#406) - [x] Events for RoomDelegate - [x] Events for ParticipantDelegate - [ ] State management --- .../LiveKit/Core/Room+EngineDelegate.swift | 28 ++++++++ Sources/LiveKit/Core/Room.swift | 1 + .../Protocols/ParticipantDelegate.swift | 4 ++ Sources/LiveKit/Protocols/RoomDelegate.swift | 4 ++ .../LiveKit/Types/TranscriptionSegment.swift | 68 +++++++++++++++++++ 5 files changed, 105 insertions(+) create mode 100644 Sources/LiveKit/Types/TranscriptionSegment.swift diff --git a/Sources/LiveKit/Core/Room+EngineDelegate.swift b/Sources/LiveKit/Core/Room+EngineDelegate.swift index ca67dde96..b1bb35b1e 100644 --- a/Sources/LiveKit/Core/Room+EngineDelegate.swift +++ b/Sources/LiveKit/Core/Room+EngineDelegate.swift @@ -194,4 +194,32 @@ extension Room { } } } + + func room(didReceiveTranscriptionPacket packet: Livekit_Transcription) { + // Try to find matching Participant. + guard let participant = allParticipants[Participant.Identity(from: packet.transcribedParticipantIdentity)] else { + log("[Transcription] Could not find participant: \(packet.transcribedParticipantIdentity)", .warning) + return + } + + guard let publication = participant._state.read({ $0.trackPublications[Track.Sid(from: packet.trackID)] }) else { + log("[Transcription] Could not find publication: \(packet.trackID)", .warning) + return + } + + let segments = packet.segments.map { $0.toLKType() } + + guard !segments.isEmpty else { + log("[Transcription] Received segments are empty", .warning) + return + } + + delegates.notify { + $0.room?(self, participant: participant, trackPublication: publication, didReceiveTranscriptionSegments: segments) + } + + participant.delegates.notify { + $0.participant?(participant, trackPublication: publication, didReceiveTranscriptionSegments: segments) + } + } } diff --git a/Sources/LiveKit/Core/Room.swift b/Sources/LiveKit/Core/Room.swift index 18f8aefb4..59e7673d7 100644 --- a/Sources/LiveKit/Core/Room.swift +++ b/Sources/LiveKit/Core/Room.swift @@ -521,6 +521,7 @@ extension Room: DataChannelDelegate { switch dataPacket.value { case let .speaker(update): engine(self, didUpdateSpeakers: update.speakers) case let .user(userPacket): engine(self, didReceiveUserPacket: userPacket) + case let .transcription(packet): room(didReceiveTranscriptionPacket: packet) default: return } } diff --git a/Sources/LiveKit/Protocols/ParticipantDelegate.swift b/Sources/LiveKit/Protocols/ParticipantDelegate.swift index 269973076..86a982560 100644 --- a/Sources/LiveKit/Protocols/ParticipantDelegate.swift +++ b/Sources/LiveKit/Protocols/ParticipantDelegate.swift @@ -65,6 +65,10 @@ public protocol ParticipantDelegate: AnyObject { @objc optional func participant(_ participant: Participant, trackPublication: TrackPublication, didUpdateIsMuted isMuted: Bool) + /// Received transcription segments. + @objc optional + func participant(_ participant: Participant, trackPublication: TrackPublication, didReceiveTranscriptionSegments segments: [TranscriptionSegment]) + // MARK: - LocalTrackPublication /// The ``LocalParticipant`` has published a ``LocalTrackPublication``. diff --git a/Sources/LiveKit/Protocols/RoomDelegate.swift b/Sources/LiveKit/Protocols/RoomDelegate.swift index ac8a1bd10..5e1b04183 100644 --- a/Sources/LiveKit/Protocols/RoomDelegate.swift +++ b/Sources/LiveKit/Protocols/RoomDelegate.swift @@ -102,6 +102,10 @@ public protocol RoomDelegate: AnyObject { @objc optional func room(_ room: Room, participant: Participant, didUpdateAttributes attributes: [String: String]) + /// Received transcription segments. + @objc optional + func room(_ room: Room, participant: Participant, trackPublication: TrackPublication, didReceiveTranscriptionSegments segments: [TranscriptionSegment]) + // MARK: - Track Publications /// The ``LocalParticipant`` has published a ``LocalTrack``. diff --git a/Sources/LiveKit/Types/TranscriptionSegment.swift b/Sources/LiveKit/Types/TranscriptionSegment.swift new file mode 100644 index 000000000..b2eef7984 --- /dev/null +++ b/Sources/LiveKit/Types/TranscriptionSegment.swift @@ -0,0 +1,68 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +@objc +public class TranscriptionSegment: NSObject { + public let id: String + public let text: String + public let language: String + public let startTime: UInt64 + public let endTime: UInt64 + public let isFinal: Bool + + init(id: String, + text: String, + language: String, + startTime: UInt64, + endTime: UInt64, + isFinal: Bool) + { + self.id = id + self.text = text + self.language = language + self.startTime = startTime + self.endTime = endTime + self.isFinal = isFinal + } + + // MARK: - Equal + + override public func isEqual(_ object: Any?) -> Bool { + guard let other = object as? Self else { return false } + return id == other.id + } + + override public var hash: Int { + var hasher = Hasher() + hasher.combine(id) + return hasher.finalize() + } +} + +// MARK: - Internal + +extension Livekit_TranscriptionSegment { + func toLKType() -> TranscriptionSegment { + TranscriptionSegment(id: id, + text: text, + language: language, + startTime: startTime, + endTime: endTime, + isFinal: final) + } +}