From 91bf10ce18c87e60cfae294a9f2bc00aea6b9f7a Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Sun, 8 Dec 2024 21:45:26 +0800 Subject: [PATCH] feat: audio visualizer. (#638) * [wip] feat: audio visualizer. * macos. * wip. * wip. * wip. * wip. * wip. * update. * update. * update. * cleanup. * bump version for flutter-webrtc. * dart format. * add third-party code license. * fix. * fix compiler warning. * license. * add enableVisualizer to RoomOptions. * Added comments for enableVisualizer. --- .../io/livekit/plugin/FFTAudioAnalyzer.kt | 215 ++++++++++++++++++ .../kotlin/io/livekit/plugin/LKAudioTrack.kt | 28 +++ .../io/livekit/plugin/LKLocalAudioTrack.kt | 40 ++++ .../io/livekit/plugin/LKRemoteAudioTrack.kt | 42 ++++ .../kotlin/io/livekit/plugin/LiveKitPlugin.kt | 67 ++++++ .../kotlin/io/livekit/plugin/Visualizer.kt | 208 +++++++++++++++++ example/lib/pages/prejoin.dart | 10 +- example/lib/widgets/participant.dart | 16 +- example/lib/widgets/sound_waveform.dart | 141 ++++++++++++ ios/Classes/AVAudioPCMBuffer.swift | 1 + ios/Classes/AudioProcessing.swift | 1 + ios/Classes/AudioTrack.swift | 1 + ios/Classes/FFTProcessor.swift | 1 + ios/Classes/LocalAudioTrack.swift | 1 + ios/Classes/RemoteAudioTrack.swift | 1 + ios/Classes/RingBuffer.swift | 1 + ios/Classes/Track.swift | 1 + ios/Classes/Visualizer.swift | 1 + ios/livekit_client.podspec | 1 + lib/src/core/room.dart | 1 + lib/src/events.dart | 13 ++ lib/src/options.dart | 9 + lib/src/participant/local.dart | 3 +- lib/src/participant/remote.dart | 5 +- lib/src/support/native.dart | 36 +++ lib/src/track/local/audio.dart | 8 +- lib/src/track/local/local.dart | 56 ++++- lib/src/track/remote/audio.dart | 3 +- lib/src/track/remote/remote.dart | 3 +- lib/src/track/track.dart | 15 +- macos/Classes/AVAudioPCMBuffer.swift | 1 + macos/Classes/AudioProcessing.swift | 1 + macos/Classes/AudioTrack.swift | 1 + macos/Classes/FFTProcessor.swift | 1 + macos/Classes/LocalAudioTrack.swift | 1 + macos/Classes/RemoteAudioTrack.swift | 1 + macos/Classes/RingBuffer.swift | 1 + macos/Classes/Track.swift | 1 + macos/Classes/Visualizer.swift | 1 + macos/livekit_client.podspec | 1 + pubspec.yaml | 4 +- shared_swift/AVAudioPCMBuffer.swift | 136 +++++++++++ shared_swift/AudioProcessing.swift | 163 +++++++++++++ shared_swift/AudioTrack.swift | 28 +++ shared_swift/FFTProcessor.swift | 147 ++++++++++++ shared_swift/LiveKitPlugin.swift | 56 +++++ shared_swift/LocalAudioTrack.swift | 39 ++++ shared_swift/RemoteAudioTrack.swift | 39 ++++ shared_swift/RingBuffer.swift | 51 +++++ shared_swift/Track.swift | 30 +++ shared_swift/Visualizer.swift | 132 +++++++++++ 51 files changed, 1747 insertions(+), 17 deletions(-) create mode 100644 android/src/main/kotlin/io/livekit/plugin/FFTAudioAnalyzer.kt create mode 100644 android/src/main/kotlin/io/livekit/plugin/LKAudioTrack.kt create mode 100644 android/src/main/kotlin/io/livekit/plugin/LKLocalAudioTrack.kt create mode 100644 android/src/main/kotlin/io/livekit/plugin/LKRemoteAudioTrack.kt create mode 100644 android/src/main/kotlin/io/livekit/plugin/Visualizer.kt create mode 100644 example/lib/widgets/sound_waveform.dart create mode 120000 ios/Classes/AVAudioPCMBuffer.swift create mode 120000 ios/Classes/AudioProcessing.swift create mode 120000 ios/Classes/AudioTrack.swift create mode 120000 ios/Classes/FFTProcessor.swift create mode 120000 ios/Classes/LocalAudioTrack.swift create mode 120000 ios/Classes/RemoteAudioTrack.swift create mode 120000 ios/Classes/RingBuffer.swift create mode 120000 ios/Classes/Track.swift create mode 120000 ios/Classes/Visualizer.swift create mode 120000 macos/Classes/AVAudioPCMBuffer.swift create mode 120000 macos/Classes/AudioProcessing.swift create mode 120000 macos/Classes/AudioTrack.swift create mode 120000 macos/Classes/FFTProcessor.swift create mode 120000 macos/Classes/LocalAudioTrack.swift create mode 120000 macos/Classes/RemoteAudioTrack.swift create mode 120000 macos/Classes/RingBuffer.swift create mode 120000 macos/Classes/Track.swift create mode 120000 macos/Classes/Visualizer.swift create mode 100644 shared_swift/AVAudioPCMBuffer.swift create mode 100644 shared_swift/AudioProcessing.swift create mode 100644 shared_swift/AudioTrack.swift create mode 100755 shared_swift/FFTProcessor.swift create mode 100644 shared_swift/LocalAudioTrack.swift create mode 100644 shared_swift/RemoteAudioTrack.swift create mode 100644 shared_swift/RingBuffer.swift create mode 100644 shared_swift/Track.swift create mode 100644 shared_swift/Visualizer.swift diff --git a/android/src/main/kotlin/io/livekit/plugin/FFTAudioAnalyzer.kt b/android/src/main/kotlin/io/livekit/plugin/FFTAudioAnalyzer.kt new file mode 100644 index 00000000..11a22fc5 --- /dev/null +++ b/android/src/main/kotlin/io/livekit/plugin/FFTAudioAnalyzer.kt @@ -0,0 +1,215 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Originally adapted from: https://github.com/dzolnai/ExoVisualizer + * + * MIT License + * + * Copyright (c) 2019 Dániel Zolnai + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.livekit.plugin + +import android.media.AudioTrack +import com.paramsen.noise.Noise +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.TimeUnit +import kotlin.math.max + + +/** + * A Fast Fourier Transform analyzer for audio bytes. + * + * Use [queueInput] to add audio bytes, and collect on [fftFlow] + * to receive the analyzed frequencies. + */ +class FFTAudioAnalyzer { + + companion object { + const val SAMPLE_SIZE = 512 + private val EMPTY_BUFFER = ByteBuffer.allocateDirect(0).order(ByteOrder.nativeOrder()) + + // Extra size next in addition to the AudioTrack buffer size + private const val BUFFER_EXTRA_SIZE = SAMPLE_SIZE * 8 + + // Size of short in bytes. + private const val SHORT_SIZE = 2 + } + + val isActive: Boolean + get() = noise != null + + private var noise: Noise? = null + private lateinit var inputAudioFormat: AudioFormat + + private var audioTrackBufferSize = 0 + + private var fftBuffer: ByteBuffer = EMPTY_BUFFER + private lateinit var srcBuffer: ByteBuffer + private var srcBufferPosition = 0 + private val tempShortArray = ShortArray(SAMPLE_SIZE) + private val src = FloatArray(SAMPLE_SIZE) + + /** + * A flow of frequencies for the audio bytes given through [queueInput]. + */ + var fft: FloatArray? = null + private set + + fun configure(inputAudioFormat: AudioFormat) { + this.inputAudioFormat = inputAudioFormat + + noise = Noise.real(SAMPLE_SIZE) + + audioTrackBufferSize = getDefaultBufferSizeInBytes(inputAudioFormat) + + srcBuffer = ByteBuffer.allocate(audioTrackBufferSize + BUFFER_EXTRA_SIZE) + } + + fun release() { + noise?.close() + noise = null + } + + /** + * Add audio bytes to be processed. + */ + fun queueInput(inputBuffer: ByteBuffer) { + if (!isActive) { + return + } + var position = inputBuffer.position() + val limit = inputBuffer.limit() + val frameCount = (limit - position) / (SHORT_SIZE * inputAudioFormat.numberOfChannels) + val singleChannelOutputSize = frameCount * SHORT_SIZE + + // Setup buffer + if (fftBuffer.capacity() < singleChannelOutputSize) { + fftBuffer = + ByteBuffer.allocateDirect(singleChannelOutputSize).order(ByteOrder.nativeOrder()) + } else { + fftBuffer.clear() + } + + // Process inputBuffer + while (position < limit) { + var summedUp: Short = 0 + for (channelIndex in 0 until inputAudioFormat.numberOfChannels) { + if( channelIndex == 0) { + val current = inputBuffer.getShort(position + 2 * channelIndex) + summedUp = (summedUp + current).toShort() + } + } + fftBuffer.putShort(summedUp) + position += inputAudioFormat.numberOfChannels * 2 + } + + // Reset input buffer to original position. + inputBuffer.position(position) + + processFFT(this.fftBuffer) + } + + private fun processFFT(buffer: ByteBuffer) { + if (noise == null) { + return + } + srcBuffer.put(buffer.array()) + srcBufferPosition += buffer.array().size + // Since this is PCM 16 bit, each sample will be 2 bytes. + // So to get the sample size in the end, we need to take twice as many bytes off the buffer + val bytesToProcess = SAMPLE_SIZE * 2 + while (srcBufferPosition > bytesToProcess) { + // Move to start of + srcBuffer.position(0) + + srcBuffer.asShortBuffer().get(tempShortArray, 0, SAMPLE_SIZE) + tempShortArray.forEachIndexed { index, sample -> + // Normalize to value between -1.0 and 1.0 + src[index] = sample.toFloat() / Short.MAX_VALUE + } + + srcBuffer.position(bytesToProcess) + srcBuffer.compact() + srcBufferPosition -= bytesToProcess + srcBuffer.position(srcBufferPosition) + val dst = FloatArray(SAMPLE_SIZE + 2) + val fft = noise?.fft(src, dst)!! + + this.fft = fft + } + } + + private fun durationUsToFrames(sampleRate: Int, durationUs: Long): Long { + return durationUs * sampleRate / TimeUnit.MICROSECONDS.convert(1, TimeUnit.SECONDS) + } + + private fun getPcmFrameSize(channelCount: Int): Int { + // assumes PCM_16BIT + return channelCount * 2 + } + + private fun getAudioTrackChannelConfig(channelCount: Int): Int { + return when (channelCount) { + 1 -> android.media.AudioFormat.CHANNEL_OUT_MONO + 2 -> android.media.AudioFormat.CHANNEL_OUT_STEREO + // ignore other channel counts that aren't used in LiveKit + else -> android.media.AudioFormat.CHANNEL_INVALID + } + } + + private fun getDefaultBufferSizeInBytes(audioFormat: AudioFormat): Int { + val outputPcmFrameSize = getPcmFrameSize(audioFormat.numberOfChannels) + val minBufferSize = + AudioTrack.getMinBufferSize( + audioFormat.sampleRate, + getAudioTrackChannelConfig(audioFormat.numberOfChannels), + android.media.AudioFormat.ENCODING_PCM_16BIT + ) + + check(minBufferSize != AudioTrack.ERROR_BAD_VALUE) + val multipliedBufferSize = minBufferSize * 4 + val minAppBufferSize = + durationUsToFrames(audioFormat.sampleRate, 30 * 1000).toInt() * outputPcmFrameSize + val maxAppBufferSize = max( + minBufferSize.toLong(), + durationUsToFrames(audioFormat.sampleRate, 500 * 1000) * outputPcmFrameSize + ).toInt() + val bufferSizeInFrames = + multipliedBufferSize.coerceIn(minAppBufferSize, maxAppBufferSize) / outputPcmFrameSize + return bufferSizeInFrames * outputPcmFrameSize + } +} + +data class AudioFormat(val bitsPerSample: Int, val sampleRate: Int, val numberOfChannels: Int) \ No newline at end of file diff --git a/android/src/main/kotlin/io/livekit/plugin/LKAudioTrack.kt b/android/src/main/kotlin/io/livekit/plugin/LKAudioTrack.kt new file mode 100644 index 00000000..78357655 --- /dev/null +++ b/android/src/main/kotlin/io/livekit/plugin/LKAudioTrack.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.plugin + +import org.webrtc.AudioTrackSink + +interface LKAudioTrack { + + fun addSink(sink: AudioTrackSink?) + + fun removeSink(sink: AudioTrackSink) + + fun id(): String +} \ No newline at end of file diff --git a/android/src/main/kotlin/io/livekit/plugin/LKLocalAudioTrack.kt b/android/src/main/kotlin/io/livekit/plugin/LKLocalAudioTrack.kt new file mode 100644 index 00000000..01a6bb22 --- /dev/null +++ b/android/src/main/kotlin/io/livekit/plugin/LKLocalAudioTrack.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.plugin + +import com.cloudwebrtc.webrtc.audio.LocalAudioTrack +import org.webrtc.AudioTrackSink + +class LKLocalAudioTrack : LKAudioTrack { + private var localAudioTrack: LocalAudioTrack? = null + + constructor(localAudioTrack: LocalAudioTrack) { + this.localAudioTrack = localAudioTrack + } + + override fun addSink(sink: AudioTrackSink?) { + localAudioTrack?.addSink(sink) + } + + override fun removeSink(sink: AudioTrackSink) { + localAudioTrack?.removeSink(sink) + } + + override fun id(): String { + return localAudioTrack?.id() ?: "" + } +} \ No newline at end of file diff --git a/android/src/main/kotlin/io/livekit/plugin/LKRemoteAudioTrack.kt b/android/src/main/kotlin/io/livekit/plugin/LKRemoteAudioTrack.kt new file mode 100644 index 00000000..f165f7e3 --- /dev/null +++ b/android/src/main/kotlin/io/livekit/plugin/LKRemoteAudioTrack.kt @@ -0,0 +1,42 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.plugin + +import org.webrtc.AudioTrack +import org.webrtc.AudioTrackSink + +class LKRemoteAudioTrack: LKAudioTrack { + private var audioTrack: AudioTrack? = null + + constructor( + audioTrack: AudioTrack + ) { + this.audioTrack = audioTrack + } + + override fun addSink(sink: AudioTrackSink?) { + audioTrack?.addSink(sink) + } + + override fun removeSink(sink: AudioTrackSink) { + audioTrack?.removeSink(sink) + } + + override fun id(): String { + return audioTrack?.id() ?: "" + } +} \ No newline at end of file diff --git a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt index 56dd96ab..46f82074 100644 --- a/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt +++ b/android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt @@ -16,6 +16,7 @@ package io.livekit.plugin +import android.annotation.SuppressLint import androidx.annotation.NonNull import io.flutter.embedding.engine.plugins.FlutterPlugin @@ -24,8 +25,16 @@ import io.flutter.plugin.common.MethodChannel import io.flutter.plugin.common.MethodChannel.MethodCallHandler import io.flutter.plugin.common.MethodChannel.Result +import com.cloudwebrtc.webrtc.FlutterWebRTCPlugin +import com.cloudwebrtc.webrtc.audio.LocalAudioTrack +import io.flutter.plugin.common.BinaryMessenger +import org.webrtc.AudioTrack + /** LiveKitPlugin */ class LiveKitPlugin: FlutterPlugin, MethodCallHandler { + private var processors = mutableMapOf() + private var flutterWebRTCPlugin = FlutterWebRTCPlugin.sharedSingleton + private var binaryMessenger: BinaryMessenger? = null /// The MethodChannel that will the communication between Flutter and native Android /// /// This local reference serves to register the plugin with the Flutter Engine and unregister it @@ -35,9 +44,67 @@ class LiveKitPlugin: FlutterPlugin, MethodCallHandler { override fun onAttachedToEngine(@NonNull flutterPluginBinding: FlutterPlugin.FlutterPluginBinding) { channel = MethodChannel(flutterPluginBinding.binaryMessenger, "livekit_client") channel.setMethodCallHandler(this) + binaryMessenger = flutterPluginBinding.binaryMessenger + } + + @SuppressLint("SuspiciousIndentation") + private fun handleStartVisualizer(@NonNull call: MethodCall, @NonNull result: Result) { + val trackId = call.argument("trackId") + if (trackId == null) { + result.error("INVALID_ARGUMENT", "trackId is required", null) + return + } + var audioTrack: LKAudioTrack? = null + val barCount = call.argument("barCount") ?: 7 + val isCentered = call.argument("isCentered") ?: true + + val track = flutterWebRTCPlugin.getLocalTrack(trackId) + if (track != null) { + audioTrack = LKLocalAudioTrack(track as LocalAudioTrack) + } else { + val remoteTrack = flutterWebRTCPlugin.getRemoteTrack(trackId) + if (remoteTrack != null) { + audioTrack = LKRemoteAudioTrack(remoteTrack as AudioTrack) + } + } + + if(audioTrack == null) { + result.error("INVALID_ARGUMENT", "track not found", null) + return + } + + val visualizer = Visualizer( + barCount = barCount, isCentered = isCentered, + audioTrack = audioTrack, binaryMessenger = binaryMessenger!!) + + processors[audioTrack] = visualizer + result.success(null) + } + + private fun handleStopVisualizer(@NonNull call: MethodCall, @NonNull result: Result) { + val trackId = call.argument("trackId") + if (trackId == null) { + result.error("INVALID_ARGUMENT", "trackId is required", null) + return + } + processors.entries.removeAll { (k, v) -> + if (k.id() == trackId) { + v.stop() + true + } + false + } + result.success(null) } override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) { + if(call.method == "startVisualizer") { + handleStartVisualizer(call, result) + return + } else if(call.method == "stopVisualizer") { + handleStopVisualizer(call, result) + return + } // no-op for now result.notImplemented() } diff --git a/android/src/main/kotlin/io/livekit/plugin/Visualizer.kt b/android/src/main/kotlin/io/livekit/plugin/Visualizer.kt new file mode 100644 index 00000000..a7695b1d --- /dev/null +++ b/android/src/main/kotlin/io/livekit/plugin/Visualizer.kt @@ -0,0 +1,208 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.livekit.plugin + +import android.os.Handler +import android.os.Looper +import io.flutter.plugin.common.BinaryMessenger +import io.flutter.plugin.common.EventChannel +import org.webrtc.AudioTrack +import org.webrtc.AudioTrackSink +import java.nio.ByteBuffer +import kotlin.math.pow +import kotlin.math.round +import kotlin.math.sqrt + +class Visualizer : EventChannel.StreamHandler, AudioTrackSink { + private var eventChannel: EventChannel? = null + private var eventSink: EventChannel.EventSink? = null + private var ffiAudioAnalyzer = FFTAudioAnalyzer() + private var audioTrack: LKAudioTrack? = null + + private var amplitudes: FloatArray = FloatArray(0) + private var bands: FloatArray = FloatArray(7) + + private var barCount: Int = 7 + private var loPass: Int = 1 + private var hiPass: Int = 120 + private var isCentered: Boolean = true + + private var audioFormat = AudioFormat(16, 48000, 1) + + constructor( + barCount: Int, + isCentered: Boolean, + audioTrack: LKAudioTrack, + binaryMessenger: BinaryMessenger) { + this.barCount = barCount + this.isCentered = isCentered + this.audioTrack = audioTrack + eventChannel = EventChannel(binaryMessenger, "io.livekit.audio.visualizer/eventChannel-" + audioTrack.id()) + eventChannel?.setStreamHandler(this) + ffiAudioAnalyzer.configure(audioFormat) + audioTrack.addSink(this) + } + + fun stop() { + audioTrack?.removeSink(this) + eventChannel?.setStreamHandler(null) + ffiAudioAnalyzer.release() + } + + override fun onData( + audioData: ByteBuffer, + bitsPerSample: Int, + sampleRate: Int, + numberOfChannels: Int, + numberOfFrames: Int, + absoluteCaptureTimestampMs: Long + ) { + + if (audioFormat.sampleRate != sampleRate || audioFormat.bitsPerSample != bitsPerSample || audioFormat.numberOfChannels != numberOfChannels) { + audioFormat = AudioFormat(bitsPerSample, sampleRate, numberOfChannels) + ffiAudioAnalyzer.configure(audioFormat) + } + + ffiAudioAnalyzer.queueInput(audioData) + var fft: FloatArray? = ffiAudioAnalyzer.fft ?: return + + + + val averages = FloatArray(barCount) + + var sliced = fft?.slice(loPass until hiPass) + amplitudes = sliced?.let { calculateAmplitudeBarsFromFFT(it, averages, barCount) }!! + + amplitudes = centerBands(amplitudes) + + bands = bands.mapIndexed { index, value -> + smoothTransition(value, amplitudes[index], 0.3f) + }.toFloatArray() + + handler.post { + eventSink?.success(bands) + } + } + + private val handler: Handler by lazy { + Handler(Looper.getMainLooper()) + } + + override fun onListen(arguments: Any?, events: EventChannel.EventSink?) { + eventSink = events + } + + override fun onCancel(arguments: Any?) { + eventSink = null + } +} + +private fun centerBands(bands: FloatArray): FloatArray { + var centeredBands = FloatArray(bands.size) + var leftIndex = bands.size / 2; + var rightIndex = leftIndex; + + for (i in bands.indices) { + val value = bands[i] + if (i % 2 == 0) { + // Place value to the right + centeredBands[rightIndex] = value + rightIndex += 1 + } else { + // Place value to the left + leftIndex -= 1 + centeredBands[leftIndex] = value + } + } + return centeredBands +} + +private fun smoothTransition(from: Float, to: Float, factor: Float): Float { + + val delta = to - from + val easedFactor = easeInOutCubic(factor) + return from + delta * easedFactor +} + +private fun easeInOutCubic(t: Float): Float { + return if (t < 0.5) { + 4 * t * t * t + } else { + 1 - (2 * t - 2).pow(3) / 2 + } +} + +private const val MIN_CONST = 0.1f +private const val MAX_CONST = 1.0f + +private fun calculateAmplitudeBarsFromFFT( + fft: List, + averages: FloatArray, + barCount: Int, +): FloatArray { + val amplitudes = FloatArray(barCount) + if (fft.isEmpty()) { + return amplitudes + } + + // We average out the values over 3 occurences (plus the current one), so big jumps are smoothed out + // Iterate over the entire FFT result array. + for (barIndex in 0 until barCount) { + // Note: each FFT is a real and imaginary pair. + // Scale down by 2 and scale back up to ensure we get an even number. + val prevLimit = (round(fft.size.toFloat() / 2 * barIndex / barCount).toInt() * 2) + .coerceIn(0, fft.size - 1) + val nextLimit = (round(fft.size.toFloat() / 2 * (barIndex + 1) / barCount).toInt() * 2) + .coerceIn(0, fft.size - 1) + + var accum = 0f + // Here we iterate within this single band + for (i in prevLimit until nextLimit step 2) { + // Convert real and imaginary part to get energy + + val realSq = fft[i] + .toDouble() + .pow(2.0) + val imaginarySq = fft[i + 1] + .toDouble() + .pow(2.0) + val raw = sqrt(realSq + imaginarySq).toFloat() + + accum += raw + } + + // A window might be empty which would result in a 0 division + if ((nextLimit - prevLimit) != 0) { + accum /= (nextLimit - prevLimit) + } else { + accum = 0.0f + } + + val smoothingFactor = 5 + var avg = averages[barIndex] + avg += (accum - avg / smoothingFactor) + averages[barIndex] = avg + + var amplitude = avg.coerceIn(MIN_CONST, MAX_CONST) + amplitude -= MIN_CONST + amplitude /= (MAX_CONST - MIN_CONST) + amplitudes[barIndex] = amplitude + } + + return amplitudes +} + diff --git a/example/lib/pages/prejoin.dart b/example/lib/pages/prejoin.dart index fae8c9d7..f600f862 100644 --- a/example/lib/pages/prejoin.dart +++ b/example/lib/pages/prejoin.dart @@ -127,9 +127,12 @@ class _PreJoinPageState extends State { } if (_selectedAudioDevice != null) { - _audioTrack = await LocalAudioTrack.create(AudioCaptureOptions( - deviceId: _selectedAudioDevice!.deviceId, - )); + _audioTrack = await LocalAudioTrack.create( + AudioCaptureOptions( + deviceId: _selectedAudioDevice!.deviceId, + ), + true, // enableVisualizer + ); await _audioTrack!.start(); } } @@ -209,6 +212,7 @@ class _PreJoinPageState extends State { screenShareEncoding: screenEncoding, ), e2eeOptions: e2eeOptions, + enableVisualizer: true, ), ); // Create a Listener before connecting diff --git a/example/lib/widgets/participant.dart b/example/lib/widgets/participant.dart index 4becd66a..1e090646 100644 --- a/example/lib/widgets/participant.dart +++ b/example/lib/widgets/participant.dart @@ -7,6 +7,7 @@ import 'package:livekit_example/theme.dart'; import 'no_video.dart'; import 'participant_info.dart'; import 'participant_stats.dart'; +import 'sound_waveform.dart'; abstract class ParticipantWidget extends StatefulWidget { // Convenience method to return relevant widget for participant @@ -175,7 +176,19 @@ abstract class _ParticipantWidgetState right: 30, child: ParticipantStatsWidget( participant: widget.participant, - )), + )), + if(activeAudioTrack != null && !activeAudioTrack!.muted) Positioned( + top: 10, + right: 10, + left: 10, + bottom: 10, + child: SoundWaveformWidget( + key: ValueKey(activeAudioTrack!.hashCode), + audioTrack: activeAudioTrack!, + width: 8, + ), + ), + ], ), ); @@ -360,3 +373,4 @@ class RemoteTrackQualityMenuWidget extends StatelessWidget { ), ); } + diff --git a/example/lib/widgets/sound_waveform.dart b/example/lib/widgets/sound_waveform.dart new file mode 100644 index 00000000..0ac81eef --- /dev/null +++ b/example/lib/widgets/sound_waveform.dart @@ -0,0 +1,141 @@ +/* + * Copyright 2024 LiveKit, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * [SoundWaveformWidget] Originally adapted from: https://github.com/SushanShakya/sound_waveform + * + * MIT License + * + * Copyright (c) 2022 Sushan Shakya + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import 'package:flutter/material.dart'; +import 'package:livekit_client/livekit_client.dart'; + +class SoundWaveformWidget extends StatefulWidget { + final int count; + final double width; + final double minHeight; + final double maxHeight; + final int durationInMilliseconds; + const SoundWaveformWidget({ + super.key, + required this.audioTrack, + this.count = 7, + this.width = 5, + this.minHeight = 8, + this.maxHeight = 100, + this.durationInMilliseconds = 500, + }); + final AudioTrack audioTrack; + @override + State createState() => _SoundWaveformWidgetState(); +} + +class _SoundWaveformWidgetState extends State + with TickerProviderStateMixin { + late AnimationController controller; + List samples = [0, 0, 0, 0, 0, 0, 0]; + EventsListener? _listener; + + void _startVisualizer(AudioTrack track) async { + await _listener?.dispose(); + _listener = track.createListener(); + _listener?.on((e) { + if (mounted) { + setState(() { + samples = e.event.map((e) => ((e as num) * 100).toDouble()).toList(); + }); + } + }); + } + + void _stopVisualizer(AudioTrack track) async { + await _listener?.dispose(); + } + + @override + void initState() { + super.initState(); + + _startVisualizer(widget.audioTrack); + + controller = AnimationController( + vsync: this, + duration: Duration( + milliseconds: widget.durationInMilliseconds, + )) + ..repeat(); + } + + @override + void dispose() { + controller.dispose(); + _stopVisualizer(widget.audioTrack); + super.dispose(); + } + + @override + Widget build(BuildContext context) { + final count = widget.count; + final minHeight = widget.minHeight; + final maxHeight = widget.maxHeight; + return AnimatedBuilder( + animation: controller, + builder: (c, child) { + return Row( + mainAxisSize: MainAxisSize.min, + children: List.generate( + count, + (i) => AnimatedContainer( + duration: Duration( + milliseconds: widget.durationInMilliseconds ~/ count), + margin: i == (samples.length - 1) + ? EdgeInsets.zero + : const EdgeInsets.only(right: 5), + height: samples[i] < minHeight + ? minHeight + : samples[i] > maxHeight + ? maxHeight + : samples[i], + width: widget.width, + decoration: BoxDecoration( + color: Colors.white, + borderRadius: BorderRadius.circular(9999), + ), + ), + ), + ); + }, + ); + } +} diff --git a/ios/Classes/AVAudioPCMBuffer.swift b/ios/Classes/AVAudioPCMBuffer.swift new file mode 120000 index 00000000..0afb6483 --- /dev/null +++ b/ios/Classes/AVAudioPCMBuffer.swift @@ -0,0 +1 @@ +../../shared_swift/AVAudioPCMBuffer.swift \ No newline at end of file diff --git a/ios/Classes/AudioProcessing.swift b/ios/Classes/AudioProcessing.swift new file mode 120000 index 00000000..6e5154e9 --- /dev/null +++ b/ios/Classes/AudioProcessing.swift @@ -0,0 +1 @@ +../../shared_swift/AudioProcessing.swift \ No newline at end of file diff --git a/ios/Classes/AudioTrack.swift b/ios/Classes/AudioTrack.swift new file mode 120000 index 00000000..cc8fcabb --- /dev/null +++ b/ios/Classes/AudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/AudioTrack.swift \ No newline at end of file diff --git a/ios/Classes/FFTProcessor.swift b/ios/Classes/FFTProcessor.swift new file mode 120000 index 00000000..6be4294e --- /dev/null +++ b/ios/Classes/FFTProcessor.swift @@ -0,0 +1 @@ +../../shared_swift/FFTProcessor.swift \ No newline at end of file diff --git a/ios/Classes/LocalAudioTrack.swift b/ios/Classes/LocalAudioTrack.swift new file mode 120000 index 00000000..1c654654 --- /dev/null +++ b/ios/Classes/LocalAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/LocalAudioTrack.swift \ No newline at end of file diff --git a/ios/Classes/RemoteAudioTrack.swift b/ios/Classes/RemoteAudioTrack.swift new file mode 120000 index 00000000..c320b7d1 --- /dev/null +++ b/ios/Classes/RemoteAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/RemoteAudioTrack.swift \ No newline at end of file diff --git a/ios/Classes/RingBuffer.swift b/ios/Classes/RingBuffer.swift new file mode 120000 index 00000000..3eb311db --- /dev/null +++ b/ios/Classes/RingBuffer.swift @@ -0,0 +1 @@ +../../shared_swift/RingBuffer.swift \ No newline at end of file diff --git a/ios/Classes/Track.swift b/ios/Classes/Track.swift new file mode 120000 index 00000000..8da7267a --- /dev/null +++ b/ios/Classes/Track.swift @@ -0,0 +1 @@ +../../shared_swift/Track.swift \ No newline at end of file diff --git a/ios/Classes/Visualizer.swift b/ios/Classes/Visualizer.swift new file mode 120000 index 00000000..0152352b --- /dev/null +++ b/ios/Classes/Visualizer.swift @@ -0,0 +1 @@ +../../shared_swift/Visualizer.swift \ No newline at end of file diff --git a/ios/livekit_client.podspec b/ios/livekit_client.podspec index b8d3ec1e..05b22f53 100644 --- a/ios/livekit_client.podspec +++ b/ios/livekit_client.podspec @@ -17,4 +17,5 @@ Pod::Spec.new do |s| s.dependency 'Flutter' s.dependency 'WebRTC-SDK', '125.6422.06' + s.dependency 'flutter_webrtc' end diff --git a/lib/src/core/room.dart b/lib/src/core/room.dart index 987f8bfb..bdcdb114 100644 --- a/lib/src/core/room.dart +++ b/lib/src/core/room.dart @@ -576,6 +576,7 @@ class Room extends DisposableChangeNotifier with EventsEmittable { trackSid, receiver: event.receiver, audioOutputOptions: roomOptions.defaultAudioOutputOptions, + enableVisualizer: roomOptions.enableVisualizer, ); } on TrackSubscriptionExceptionEvent catch (event) { logger.severe('addSubscribedMediaTrack() throwed ${event}'); diff --git a/lib/src/events.dart b/lib/src/events.dart index 04ef406e..cc193540 100644 --- a/lib/src/events.dart +++ b/lib/src/events.dart @@ -565,3 +565,16 @@ class VideoReceiverStatsEvent with TrackEvent { String toString() => '${runtimeType}' 'stats: ${stats})'; } + +class AudioVisualizerEvent with TrackEvent { + final Track track; + final List event; + const AudioVisualizerEvent({ + required this.track, + required this.event, + }); + + @override + String toString() => '${runtimeType}' + 'track: ${track})'; +} diff --git a/lib/src/options.dart b/lib/src/options.dart index 72ebd263..9c43dde3 100644 --- a/lib/src/options.dart +++ b/lib/src/options.dart @@ -115,6 +115,14 @@ class RoomOptions { /// Options for end-to-end encryption. final E2EEOptions? e2eeOptions; + /// audio visualizer is disabled by default + /// When enabled, the native layer will register an FFI audio analyzer + /// and will emit AudioVisualizerEvent events from AudioTrack. + /// You can use SoundWaveformWidget (example/lib/widgets/sound_waveform.dart) + /// to display the audio wave. Or write a custom widget to visualize the audio + /// wave. + final bool enableVisualizer; + const RoomOptions({ this.defaultCameraCaptureOptions = const CameraCaptureOptions(), this.defaultScreenShareCaptureOptions = const ScreenShareCaptureOptions(), @@ -126,6 +134,7 @@ class RoomOptions { this.dynacast = false, this.stopLocalTrackOnUnpublish = true, this.e2eeOptions, + this.enableVisualizer = false, }); RoomOptions copyWith({ diff --git a/lib/src/participant/local.dart b/lib/src/participant/local.dart index 3703d27d..9b328272 100644 --- a/lib/src/participant/local.dart +++ b/lib/src/participant/local.dart @@ -604,7 +604,8 @@ class LocalParticipant extends Participant { } else if (source == TrackSource.microphone) { AudioCaptureOptions captureOptions = audioCaptureOptions ?? room.roomOptions.defaultAudioCaptureOptions; - final track = await LocalAudioTrack.create(captureOptions); + final track = await LocalAudioTrack.create( + captureOptions, room.roomOptions.enableVisualizer); return await publishAudioTrack(track); } else if (source == TrackSource.screenShareVideo) { ScreenShareCaptureOptions captureOptions = screenShareCaptureOptions ?? diff --git a/lib/src/participant/remote.dart b/lib/src/participant/remote.dart index 9d5c9c40..52967779 100644 --- a/lib/src/participant/remote.dart +++ b/lib/src/participant/remote.dart @@ -112,6 +112,7 @@ class RemoteParticipant extends Participant { String trackSid, { rtc.RTCRtpReceiver? receiver, AudioOutputOptions audioOutputOptions = const AudioOutputOptions(), + bool? enableVisualizer, }) async { logger.fine('addSubscribedMediaTrack()'); @@ -153,8 +154,8 @@ class RemoteParticipant extends Participant { RemoteVideoTrack(pub.source, stream, mediaTrack, receiver: receiver); } else if (pub.kind == TrackType.AUDIO) { // audio track - track = - RemoteAudioTrack(pub.source, stream, mediaTrack, receiver: receiver); + track = RemoteAudioTrack(pub.source, stream, mediaTrack, + receiver: receiver, enableVisualizer: enableVisualizer); var listener = track.createListener(); listener.on((event) { diff --git a/lib/src/support/native.dart b/lib/src/support/native.dart index 222bdda2..e1b8a652 100644 --- a/lib/src/support/native.dart +++ b/lib/src/support/native.dart @@ -39,6 +39,42 @@ class Native { } } + @internal + static Future startVisualizer( + String trackId, { + bool isCentered = true, + int barCount = 7, + }) async { + try { + final result = await channel.invokeMethod( + 'startVisualizer', + { + 'trackId': trackId, + 'isCentered': isCentered, + 'barCount': barCount, + }, + ); + return result == true; + } catch (error) { + logger.warning('startVisualizer did throw $error'); + return false; + } + } + + @internal + static Future stopVisualizer(String trackId) async { + try { + await channel.invokeMethod( + 'stopVisualizer', + { + 'trackId': trackId, + }, + ); + } catch (error) { + logger.warning('stopVisualizer did throw $error'); + } + } + /// Returns OS's version as a string /// Currently only for iOS, macOS @internal diff --git a/lib/src/track/local/audio.dart b/lib/src/track/local/audio.dart index abc70773..90b91dd5 100644 --- a/lib/src/track/local/audio.dart +++ b/lib/src/track/local/audio.dart @@ -118,17 +118,20 @@ class LocalAudioTrack extends LocalTrack TrackSource source, rtc.MediaStream stream, rtc.MediaStreamTrack track, - this.currentOptions, - ) : super( + this.currentOptions, { + bool? enableVisualizer, + }) : super( TrackType.AUDIO, source, stream, track, + enableVisualizer: enableVisualizer, ); /// Creates a new audio track from the default audio input device. static Future create([ AudioCaptureOptions? options, + bool? enableVisualizer, ]) async { options ??= const AudioCaptureOptions(); final stream = await LocalTrack.createStream(options); @@ -138,6 +141,7 @@ class LocalAudioTrack extends LocalTrack stream, stream.getAudioTracks().first, options, + enableVisualizer: enableVisualizer, ); } } diff --git a/lib/src/track/local/local.dart b/lib/src/track/local/local.dart index 3e317715..b64c62e8 100644 --- a/lib/src/track/local/local.dart +++ b/lib/src/track/local/local.dart @@ -16,6 +16,7 @@ import 'dart:async'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:flutter/services.dart'; import 'package:flutter_webrtc/flutter_webrtc.dart' as rtc; import 'package:meta/meta.dart'; @@ -26,6 +27,7 @@ import '../../extensions.dart'; import '../../internal/events.dart'; import '../../logger.dart'; import '../../participant/remote.dart'; +import '../../support/native.dart'; import '../../support/platform.dart'; import '../../types/other.dart'; import '../options.dart'; @@ -57,7 +59,53 @@ mixin VideoTrack on Track { } /// Used to group [LocalAudioTrack] and [RemoteAudioTrack]. -mixin AudioTrack on Track {} +mixin AudioTrack on Track { + EventChannel? _eventChannel; + StreamSubscription? _streamSubscription; + + @override + Future onStarted() async { + if (enableVisualizer == true) { + await startVisualizer(); + } + } + + @override + Future onStopped() async { + if (enableVisualizer == true) { + await stopVisualizer(); + } + } + + Future startVisualizer() async { + if (_eventChannel != null) { + return; + } + + await Native.startVisualizer(mediaStreamTrack.id!); + + _eventChannel = EventChannel( + 'io.livekit.audio.visualizer/eventChannel-${mediaStreamTrack.id}'); + _streamSubscription = + _eventChannel?.receiveBroadcastStream().listen((event) { + //logger.fine('[$objectId] visualizer event(${event})'); + events.emit(AudioVisualizerEvent( + track: this, + event: event, + )); + }); + } + + Future stopVisualizer() async { + if (_eventChannel == null) { + return; + } + await Native.stopVisualizer(mediaStreamTrack.id!); + await _streamSubscription?.cancel(); + _streamSubscription = null; + _eventChannel = null; + } +} /// Base class for [LocalAudioTrack] and [LocalVideoTrack]. abstract class LocalTrack extends Track { @@ -75,12 +123,14 @@ abstract class LocalTrack extends Track { TrackType kind, TrackSource source, rtc.MediaStream mediaStream, - rtc.MediaStreamTrack mediaStreamTrack, - ) : super( + rtc.MediaStreamTrack mediaStreamTrack, { + bool? enableVisualizer, + }) : super( kind, source, mediaStream, mediaStreamTrack, + enableVisualizer: enableVisualizer, ) { mediaStreamTrack.onEnded = () { logger.fine('MediaStreamTrack.onEnded()'); diff --git a/lib/src/track/remote/audio.dart b/lib/src/track/remote/audio.dart index 184bbcb5..844c93a8 100644 --- a/lib/src/track/remote/audio.dart +++ b/lib/src/track/remote/audio.dart @@ -33,13 +33,14 @@ class RemoteAudioTrack extends RemoteTrack String? _deviceId; RemoteAudioTrack( TrackSource source, rtc.MediaStream stream, rtc.MediaStreamTrack track, - {rtc.RTCRtpReceiver? receiver}) + {rtc.RTCRtpReceiver? receiver, bool? enableVisualizer}) : super( TrackType.AUDIO, source, stream, track, receiver: receiver, + enableVisualizer: enableVisualizer, ); @override diff --git a/lib/src/track/remote/remote.dart b/lib/src/track/remote/remote.dart index dbaa0c16..03a4e45b 100644 --- a/lib/src/track/remote/remote.dart +++ b/lib/src/track/remote/remote.dart @@ -22,13 +22,14 @@ import '../track.dart'; abstract class RemoteTrack extends Track { RemoteTrack(TrackType kind, TrackSource source, rtc.MediaStream stream, rtc.MediaStreamTrack track, - {rtc.RTCRtpReceiver? receiver}) + {rtc.RTCRtpReceiver? receiver, bool? enableVisualizer}) : super( kind, source, stream, track, receiver: receiver, + enableVisualizer: enableVisualizer, ); @override diff --git a/lib/src/track/track.dart b/lib/src/track/track.dart index ee4ca53f..cbddaabc 100644 --- a/lib/src/track/track.dart +++ b/lib/src/track/track.dart @@ -59,8 +59,10 @@ abstract class Track extends DisposableChangeNotifier rtc.RTCRtpReceiver? receiver; + final bool? enableVisualizer; + Track(this.kind, this.source, this._mediaStream, this._mediaStreamTrack, - {this.receiver}) { + {this.receiver, this.enableVisualizer}) { // Any event emitted will trigger ChangeNotifier events.listen((event) { logger.finer('[TrackEvent] $event, will notifyListeners()'); @@ -110,6 +112,8 @@ abstract class Track extends DisposableChangeNotifier startMonitor(); + await onStarted(); + _active = true; return true; } @@ -125,6 +129,8 @@ abstract class Track extends DisposableChangeNotifier stopMonitor(); + await onStopped(); + logger.fine('$objectId.stop()'); _active = false; @@ -158,8 +164,15 @@ abstract class Track extends DisposableChangeNotifier Timer? _monitorTimer; + @internal Future monitorStats(); + @internal + Future onStarted() async {} + + @internal + Future onStopped() async {} + @internal void startMonitor() { _monitorTimer ??= Timer.periodic( diff --git a/macos/Classes/AVAudioPCMBuffer.swift b/macos/Classes/AVAudioPCMBuffer.swift new file mode 120000 index 00000000..0afb6483 --- /dev/null +++ b/macos/Classes/AVAudioPCMBuffer.swift @@ -0,0 +1 @@ +../../shared_swift/AVAudioPCMBuffer.swift \ No newline at end of file diff --git a/macos/Classes/AudioProcessing.swift b/macos/Classes/AudioProcessing.swift new file mode 120000 index 00000000..6e5154e9 --- /dev/null +++ b/macos/Classes/AudioProcessing.swift @@ -0,0 +1 @@ +../../shared_swift/AudioProcessing.swift \ No newline at end of file diff --git a/macos/Classes/AudioTrack.swift b/macos/Classes/AudioTrack.swift new file mode 120000 index 00000000..cc8fcabb --- /dev/null +++ b/macos/Classes/AudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/AudioTrack.swift \ No newline at end of file diff --git a/macos/Classes/FFTProcessor.swift b/macos/Classes/FFTProcessor.swift new file mode 120000 index 00000000..6be4294e --- /dev/null +++ b/macos/Classes/FFTProcessor.swift @@ -0,0 +1 @@ +../../shared_swift/FFTProcessor.swift \ No newline at end of file diff --git a/macos/Classes/LocalAudioTrack.swift b/macos/Classes/LocalAudioTrack.swift new file mode 120000 index 00000000..1c654654 --- /dev/null +++ b/macos/Classes/LocalAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/LocalAudioTrack.swift \ No newline at end of file diff --git a/macos/Classes/RemoteAudioTrack.swift b/macos/Classes/RemoteAudioTrack.swift new file mode 120000 index 00000000..c320b7d1 --- /dev/null +++ b/macos/Classes/RemoteAudioTrack.swift @@ -0,0 +1 @@ +../../shared_swift/RemoteAudioTrack.swift \ No newline at end of file diff --git a/macos/Classes/RingBuffer.swift b/macos/Classes/RingBuffer.swift new file mode 120000 index 00000000..3eb311db --- /dev/null +++ b/macos/Classes/RingBuffer.swift @@ -0,0 +1 @@ +../../shared_swift/RingBuffer.swift \ No newline at end of file diff --git a/macos/Classes/Track.swift b/macos/Classes/Track.swift new file mode 120000 index 00000000..8da7267a --- /dev/null +++ b/macos/Classes/Track.swift @@ -0,0 +1 @@ +../../shared_swift/Track.swift \ No newline at end of file diff --git a/macos/Classes/Visualizer.swift b/macos/Classes/Visualizer.swift new file mode 120000 index 00000000..0152352b --- /dev/null +++ b/macos/Classes/Visualizer.swift @@ -0,0 +1 @@ +../../shared_swift/Visualizer.swift \ No newline at end of file diff --git a/macos/livekit_client.podspec b/macos/livekit_client.podspec index 3c8ba441..0597a7de 100644 --- a/macos/livekit_client.podspec +++ b/macos/livekit_client.podspec @@ -17,4 +17,5 @@ Pod::Spec.new do |s| s.dependency 'FlutterMacOS' s.dependency 'WebRTC-SDK', '125.6422.06' + s.dependency 'flutter_webrtc' end diff --git a/pubspec.yaml b/pubspec.yaml index 9c41bb13..75215442 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -37,8 +37,8 @@ dependencies: uuid: '>=3.0.6' synchronized: ^3.0.0+3 protobuf: ^3.0.0 - flutter_webrtc: ^0.12.2 - device_info_plus: '>=8.0.0' + flutter_webrtc: ^0.12.3 + device_info_plus: ^11.1.1 js: '>=0.6.4' platform_detect: ^2.0.7 dart_webrtc: ^1.4.9 diff --git a/shared_swift/AVAudioPCMBuffer.swift b/shared_swift/AVAudioPCMBuffer.swift new file mode 100644 index 00000000..ce4ccab4 --- /dev/null +++ b/shared_swift/AVAudioPCMBuffer.swift @@ -0,0 +1,136 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Accelerate +import AVFoundation + +public extension AVAudioPCMBuffer { + func resample(toSampleRate targetSampleRate: Double) -> AVAudioPCMBuffer? { + let sourceFormat = format + + if sourceFormat.sampleRate == targetSampleRate { + // Already targetSampleRate. + return self + } + + // Define the source format (from the input buffer) and the target format. + guard let targetFormat = AVAudioFormat(commonFormat: sourceFormat.commonFormat, + sampleRate: targetSampleRate, + channels: sourceFormat.channelCount, + interleaved: sourceFormat.isInterleaved) + else { + print("Failed to create target format.") + return nil + } + + guard let converter = AVAudioConverter(from: sourceFormat, to: targetFormat) else { + print("Failed to create audio converter.") + return nil + } + + let capacity = targetFormat.sampleRate * Double(frameLength) / sourceFormat.sampleRate + + guard let convertedBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: AVAudioFrameCount(capacity)) else { + print("Failed to create converted buffer.") + return nil + } + + var isDone = false + let inputBlock: AVAudioConverterInputBlock = { _, outStatus in + if isDone { + outStatus.pointee = .noDataNow + return nil + } + outStatus.pointee = .haveData + isDone = true + return self + } + + var error: NSError? + let status = converter.convert(to: convertedBuffer, error: &error, withInputFrom: inputBlock) + + if status == .error { + print("Conversion failed: \(error?.localizedDescription ?? "Unknown error")") + return nil + } + + // Adjust frame length to the actual amount of data written + convertedBuffer.frameLength = convertedBuffer.frameCapacity + + return convertedBuffer + } + + /// Convert PCM buffer to specified common format. + /// Currently supports conversion from Int16 to Float32. + func convert(toCommonFormat commonFormat: AVAudioCommonFormat) -> AVAudioPCMBuffer? { + // Check if conversion is needed + guard format.commonFormat != commonFormat else { + return self + } + + // Check if the conversion is supported + guard format.commonFormat == .pcmFormatInt16, commonFormat == .pcmFormatFloat32 else { + print("Unsupported conversion: only Int16 to Float32 is supported") + return nil + } + + // Create output format + guard let outputFormat = AVAudioFormat(commonFormat: commonFormat, + sampleRate: format.sampleRate, + channels: format.channelCount, + interleaved: false) + else { + print("Failed to create output audio format") + return nil + } + + // Create output buffer + guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: outputFormat, + frameCapacity: frameCapacity) + else { + print("Failed to create output PCM buffer") + return nil + } + + outputBuffer.frameLength = frameLength + + let channelCount = Int(format.channelCount) + let frameCount = Int(frameLength) + + // Ensure the source buffer has Int16 data + guard let int16Data = int16ChannelData else { + print("Source buffer doesn't contain Int16 data") + return nil + } + + // Ensure the output buffer has Float32 data + guard let floatData = outputBuffer.floatChannelData else { + print("Failed to get float channel data from output buffer") + return nil + } + + // Convert Int16 to Float32 and normalize to [-1.0, 1.0] + let scale = Float(Int16.max) + var scalar = 1.0 / scale + + for channel in 0 ..< channelCount { + vDSP_vflt16(int16Data[channel], 1, floatData[channel], 1, vDSP_Length(frameCount)) + vDSP_vsmul(floatData[channel], 1, &scalar, floatData[channel], 1, vDSP_Length(frameCount)) + } + + return outputBuffer + } +} diff --git a/shared_swift/AudioProcessing.swift b/shared_swift/AudioProcessing.swift new file mode 100644 index 00000000..1722757c --- /dev/null +++ b/shared_swift/AudioProcessing.swift @@ -0,0 +1,163 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import WebRTC +import Accelerate +import AVFoundation +import Foundation + +public struct AudioLevel { + /// Linear Scale RMS Value + public let average: Float + public let peak: Float +} + +public extension RTCAudioBuffer { + /// Convert to AVAudioPCMBuffer Int16 format. + @objc + func toAVAudioPCMBuffer() -> AVAudioPCMBuffer? { + guard let audioFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: Double(frames * 100), + channels: AVAudioChannelCount(channels), + interleaved: false), + let pcmBuffer = AVAudioPCMBuffer(pcmFormat: audioFormat, + frameCapacity: AVAudioFrameCount(frames)) + else { return nil } + + pcmBuffer.frameLength = AVAudioFrameCount(frames) + + guard let targetBufferPointer = pcmBuffer.int16ChannelData else { return nil } + + for i in 0 ..< channels { + let sourceBuffer = rawBuffer(forChannel: i) + let targetBuffer = targetBufferPointer[i] + // sourceBuffer is in the format of [Int16] but is stored in 32-bit alignment, we need to pack the Int16 data correctly. + + for frame in 0 ..< frames { + // Cast and pack the source 32-bit Int16 data into the target 16-bit buffer + let clampedValue = max(Float(Int16.min), min(Float(Int16.max), sourceBuffer[frame])) + targetBuffer[frame] = Int16(clampedValue) + } + } + + return pcmBuffer + } +} + +public extension AVAudioPCMBuffer { + /// Computes Peak and Linear Scale RMS Value (Average) for all channels. + func audioLevels() -> [AudioLevel] { + var result: [AudioLevel] = [] + guard let data = floatChannelData else { + // Not containing float data + return result + } + + for i in 0 ..< Int(format.channelCount) { + let channelData = data[i] + var max: Float = 0.0 + vDSP_maxv(channelData, stride, &max, vDSP_Length(frameLength)) + var rms: Float = 0.0 + vDSP_rmsqv(channelData, stride, &rms, vDSP_Length(frameLength)) + + // No conversion to dB, return linear scale values directly + result.append(AudioLevel(average: rms, peak: max)) + } + + return result + } +} + +public extension Sequence where Iterator.Element == AudioLevel { + /// Combines all elements into a single audio level by computing the average value of all elements. + func combine() -> AudioLevel? { + var count = 0 + let totalSums: (averageSum: Float, peakSum: Float) = reduce((averageSum: 0.0, peakSum: 0.0)) { totals, audioLevel in + count += 1 + return (totals.averageSum + audioLevel.average, + totals.peakSum + audioLevel.peak) + } + + guard count > 0 else { return nil } + + return AudioLevel(average: totalSums.averageSum / Float(count), + peak: totalSums.peakSum / Float(count)) + } +} + +public class AudioVisualizeProcessor { + static let bufferSize = 1024 + + // MARK: - Public + + public let minFrequency: Float + public let maxFrequency: Float + public let minDB: Float + public let maxDB: Float + public let bandsCount: Int + + private var bands: [Float]? + + // MARK: - Private + + private let ringBuffer = RingBuffer(size: AudioVisualizeProcessor.bufferSize) + private let processor: FFTProcessor + + public init(minFrequency: Float = 10, + maxFrequency: Float = 8000, + minDB: Float = -32.0, + maxDB: Float = 32.0, + bandsCount: Int = 100) + { + self.minFrequency = minFrequency + self.maxFrequency = maxFrequency + self.minDB = minDB + self.maxDB = maxDB + self.bandsCount = bandsCount + + processor = FFTProcessor(bufferSize: Self.bufferSize) + bands = [Float](repeating: 0.0, count: bandsCount) + } + + public func process(pcmBuffer: AVAudioPCMBuffer) -> [Float]? { + guard let pcmBuffer = pcmBuffer.convert(toCommonFormat: .pcmFormatFloat32) else { return nil } + guard let floatChannelData = pcmBuffer.floatChannelData else { return nil } + + // Get the float array. + let floats = Array(UnsafeBufferPointer(start: floatChannelData[0], count: Int(pcmBuffer.frameLength))) + ringBuffer.write(floats) + + // Get full-size buffer if available, otherwise return + guard let buffer = ringBuffer.read() else { return nil } + + // Process FFT and compute frequency bands + let fftRes = processor.process(buffer: buffer) + let bands = fftRes.computeBands( + minFrequency: minFrequency, + maxFrequency: maxFrequency, + bandsCount: bandsCount, + sampleRate: Float(pcmBuffer.format.sampleRate) + ) + + let headroom = maxDB - minDB + + // Normalize magnitudes (already in decibels) + return bands.magnitudes.map { magnitude in + let adjustedMagnitude = max(0, magnitude + abs(minDB)) + return min(1.0, adjustedMagnitude / headroom) + } + } +} diff --git a/shared_swift/AudioTrack.swift b/shared_swift/AudioTrack.swift new file mode 100644 index 00000000..d1994652 --- /dev/null +++ b/shared_swift/AudioTrack.swift @@ -0,0 +1,28 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC + +@objc +public protocol AudioTrack where Self: Track { + + @objc(addAudioRenderer:) + func add(audioRenderer: RTCAudioRenderer) + + @objc(removeAudioRenderer:) + func remove(audioRenderer: RTCAudioRenderer) +} diff --git a/shared_swift/FFTProcessor.swift b/shared_swift/FFTProcessor.swift new file mode 100755 index 00000000..83ab75cc --- /dev/null +++ b/shared_swift/FFTProcessor.swift @@ -0,0 +1,147 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Accelerate +import AVFoundation + +extension Float { + var nyquistFrequency: Float { self / 2.0 } +} + +public struct FFTComputeBandsResult { + let count: Int + let magnitudes: [Float] + let frequencies: [Float] +} + +public class FFTResult { + public let magnitudes: [Float] + + init(magnitudes: [Float]) { + self.magnitudes = magnitudes + } + + func computeBands(minFrequency: Float, maxFrequency: Float, bandsCount: Int, sampleRate: Float) -> FFTComputeBandsResult { + let actualMaxFrequency = min(sampleRate.nyquistFrequency, maxFrequency) + var bandMagnitudes = [Float](repeating: 0.0, count: bandsCount) + var bandFrequencies = [Float](repeating: 0.0, count: bandsCount) + + let magLowerRange = _magnitudeIndex(for: minFrequency, sampleRate: sampleRate) + let magUpperRange = _magnitudeIndex(for: actualMaxFrequency, sampleRate: sampleRate) + let ratio = Float(magUpperRange - magLowerRange) / Float(bandsCount) + + return magnitudes.withUnsafeBufferPointer { magnitudesPtr in + for i in 0 ..< bandsCount { + let magsStartIdx = vDSP_Length(floorf(Float(i) * ratio)) + magLowerRange + let magsEndIdx = vDSP_Length(floorf(Float(i + 1) * ratio)) + magLowerRange + + let count = magsEndIdx - magsStartIdx + if count > 0 { + var sum: Float = 0 + vDSP_sve(magnitudesPtr.baseAddress! + Int(magsStartIdx), 1, &sum, count) + bandMagnitudes[i] = sum / Float(count) + } else { + bandMagnitudes[i] = magnitudes[Int(magsStartIdx)] + } + + // Compute average frequency + let bandwidth = sampleRate.nyquistFrequency / Float(magnitudes.count) + bandFrequencies[i] = (bandwidth * Float(magsStartIdx) + bandwidth * Float(magsEndIdx)) / 2 + } + + return FFTComputeBandsResult(count: bandsCount, magnitudes: bandMagnitudes, frequencies: bandFrequencies) + } + } + + @inline(__always) private func _magnitudeIndex(for frequency: Float, sampleRate: Float) -> vDSP_Length { + vDSP_Length(Float(magnitudes.count) * frequency / sampleRate.nyquistFrequency) + } +} + +class FFTProcessor { + public enum WindowType { + case none + case hanning + case hamming + } + + public let bufferSize: vDSP_Length + public let windowType: WindowType + + private let bufferHalfSize: vDSP_Length + private let bufferLog2Size: vDSP_Length + private var window: [Float] = [] + private var fftSetup: FFTSetup + private var realBuffer: [Float] + private var imaginaryBuffer: [Float] + private var zeroDBReference: Float = 1.0 + + init(bufferSize: Int, windowType: WindowType = .hanning) { + self.bufferSize = vDSP_Length(bufferSize) + self.windowType = windowType + + bufferHalfSize = vDSP_Length(bufferSize / 2) + bufferLog2Size = vDSP_Length(log2f(Float(bufferSize))) + + realBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + imaginaryBuffer = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + window = [Float](repeating: 1.0, count: Int(bufferSize)) + + fftSetup = vDSP_create_fftsetup(UInt(bufferLog2Size), FFTRadix(FFT_RADIX2))! + + switch windowType { + case .none: + break + case .hanning: + vDSP_hann_window(&window, vDSP_Length(bufferSize), Int32(vDSP_HANN_NORM)) + case .hamming: + vDSP_hamm_window(&window, vDSP_Length(bufferSize), 0) + } + } + + deinit { + vDSP_destroy_fftsetup(fftSetup) + } + + func process(buffer: [Float]) -> FFTResult { + precondition(buffer.count == Int(bufferSize), "Input buffer size mismatch.") + + var windowedBuffer = [Float](repeating: 0.0, count: Int(bufferSize)) + + vDSP_vmul(buffer, 1, window, 1, &windowedBuffer, 1, bufferSize) + + return realBuffer.withUnsafeMutableBufferPointer { realPtr in + imaginaryBuffer.withUnsafeMutableBufferPointer { imagPtr in + var complexBuffer = DSPSplitComplex(realp: realPtr.baseAddress!, imagp: imagPtr.baseAddress!) + + windowedBuffer.withUnsafeBufferPointer { bufferPtr in + let complexPtr = UnsafeRawPointer(bufferPtr.baseAddress!).bindMemory(to: DSPComplex.self, capacity: Int(bufferHalfSize)) + vDSP_ctoz(complexPtr, 2, &complexBuffer, 1, bufferHalfSize) + } + + vDSP_fft_zrip(fftSetup, &complexBuffer, 1, bufferLog2Size, FFTDirection(FFT_FORWARD)) + + var magnitudes = [Float](repeating: 0.0, count: Int(bufferHalfSize)) + vDSP_zvabs(&complexBuffer, 1, &magnitudes, 1, bufferHalfSize) + + // Convert magnitudes to decibels + vDSP_vdbcon(magnitudes, 1, &zeroDBReference, &magnitudes, 1, vDSP_Length(magnitudes.count), 1) + + return FFTResult(magnitudes: magnitudes) + } + } + } +} diff --git a/shared_swift/LiveKitPlugin.swift b/shared_swift/LiveKitPlugin.swift index c87d468a..c942afbd 100644 --- a/shared_swift/LiveKitPlugin.swift +++ b/shared_swift/LiveKitPlugin.swift @@ -13,6 +13,7 @@ // limitations under the License. import WebRTC +import flutter_webrtc #if os(macOS) import Cocoa @@ -24,6 +25,10 @@ import UIKit public class LiveKitPlugin: NSObject, FlutterPlugin { + var processers: Dictionary = [:] + + var binaryMessenger: FlutterBinaryMessenger? + public static func register(with registrar: FlutterPluginRegistrar) { #if os(macOS) @@ -34,6 +39,7 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { let channel = FlutterMethodChannel(name: "livekit_client", binaryMessenger: messenger) let instance = LiveKitPlugin() + instance.binaryMessenger = messenger registrar.addMethodCallDelegate(instance, channel: channel) } @@ -84,6 +90,52 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { return result } #endif + + public func handleStartAudioVisualizer(args: [String: Any?], result: @escaping FlutterResult) { + let webrtc = FlutterWebRTCPlugin.sharedSingleton() + + let trackId = args["trackId"] as? String + let barCount = args["barCount"] as? Int ?? 7 + let isCentered = args["isCentered"] as? Bool ?? true + + if let unwrappedTrackId = trackId { + + let localTrack = webrtc?.localTracks![unwrappedTrackId] + if let audioTrack = localTrack as? LocalAudioTrack { + let lkLocalTrack = LKLocalAudioTrack(name: unwrappedTrackId, track: audioTrack); + let processor = Visualizer(track: lkLocalTrack, + binaryMessenger: self.binaryMessenger!, + bandCount: barCount, + isCentered: isCentered) + processers[lkLocalTrack] = processor + } + + let track = webrtc?.remoteTrack(forId: unwrappedTrackId) + if let audioTrack = track as? RTCAudioTrack { + let lkRemoteTrack = LKRemoteAudioTrack(name: unwrappedTrackId, track: audioTrack); + let processor = Visualizer(track: lkRemoteTrack, + binaryMessenger: self.binaryMessenger!, + bandCount: barCount, + isCentered: isCentered) + processers[lkRemoteTrack] = processor + } + } + + + result(true) + } + + public func handleStopAudioVisualizer(args: [String: Any?], result: @escaping FlutterResult) { + let trackId = args["trackId"] as? String + if let unwrappedTrackId = trackId { + for key in processers.keys { + if key.mediaTrack.trackId == unwrappedTrackId { + processers.removeValue(forKey: key) + } + } + } + result(true) + } public func handleConfigureNativeAudio(args: [String: Any?], result: @escaping FlutterResult) { @@ -181,6 +233,10 @@ public class LiveKitPlugin: NSObject, FlutterPlugin { switch call.method { case "configureNativeAudio": handleConfigureNativeAudio(args: args, result: result) + case "startVisualizer": + handleStartAudioVisualizer(args: args, result: result) + case "stopVisualizer": + handleStopAudioVisualizer(args: args, result: result) case "osVersionString": result(LiveKitPlugin.osVersionString()) default: diff --git a/shared_swift/LocalAudioTrack.swift b/shared_swift/LocalAudioTrack.swift new file mode 100644 index 00000000..2a5568d8 --- /dev/null +++ b/shared_swift/LocalAudioTrack.swift @@ -0,0 +1,39 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC +import flutter_webrtc + +public class LKLocalAudioTrack: Track, AudioTrack { + let audioTrack: LocalAudioTrack + init(name: String, + track: LocalAudioTrack) + { + audioTrack = track + super.init(track: track.track()) + } +} + +public extension LKLocalAudioTrack { + func add(audioRenderer: RTCAudioRenderer) { + audioTrack.add(audioRenderer) + } + + func remove(audioRenderer: RTCAudioRenderer) { + audioTrack.remove(audioRenderer) + } +} diff --git a/shared_swift/RemoteAudioTrack.swift b/shared_swift/RemoteAudioTrack.swift new file mode 100644 index 00000000..de317542 --- /dev/null +++ b/shared_swift/RemoteAudioTrack.swift @@ -0,0 +1,39 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC +import flutter_webrtc + +public class LKRemoteAudioTrack: Track, AudioTrack { + let audioTrack: RTCAudioTrack + init(name: String, + track: RTCAudioTrack) + { + audioTrack = track + super.init(track: track) + } +} + +public extension LKRemoteAudioTrack { + func add(audioRenderer: RTCAudioRenderer) { + audioTrack.add(audioRenderer) + } + + func remove(audioRenderer: RTCAudioRenderer) { + audioTrack.remove(audioRenderer) + } +} diff --git a/shared_swift/RingBuffer.swift b/shared_swift/RingBuffer.swift new file mode 100644 index 00000000..ddbc1e68 --- /dev/null +++ b/shared_swift/RingBuffer.swift @@ -0,0 +1,51 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation + +// Simple ring-buffer used for internal audio processing. Not thread-safe. +class RingBuffer { + private var _isFull = false + private var _buffer: [T] + private var _head: Int = 0 + + init(size: Int) { + _buffer = [T](repeating: 0, count: size) + } + + func write(_ value: T) { + _buffer[_head] = value + _head = (_head + 1) % _buffer.count + if _head == 0 { _isFull = true } + } + + func write(_ sequence: [T]) { + for value in sequence { + write(value) + } + } + + func read() -> [T]? { + guard _isFull else { return nil } + + if _head == 0 { + return _buffer // Return the entire buffer if _head is at the start + } else { + // Return the buffer in the correct order + return Array(_buffer[_head ..< _buffer.count] + _buffer[0 ..< _head]) + } + } +} diff --git a/shared_swift/Track.swift b/shared_swift/Track.swift new file mode 100644 index 00000000..a149baf2 --- /dev/null +++ b/shared_swift/Track.swift @@ -0,0 +1,30 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import Foundation +import WebRTC + +@objc +public class Track: NSObject { + + let mediaTrack: RTCMediaStreamTrack + + init(track: RTCMediaStreamTrack) + { + mediaTrack = track + super.init() + } +} diff --git a/shared_swift/Visualizer.swift b/shared_swift/Visualizer.swift new file mode 100644 index 00000000..6a462877 --- /dev/null +++ b/shared_swift/Visualizer.swift @@ -0,0 +1,132 @@ +/* + * Copyright 2024 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import AVFoundation +import WebRTC + +#if os(macOS) +import Cocoa +import FlutterMacOS +#else +import Flutter +import UIKit +#endif + +public class Visualizer: NSObject, RTCAudioRenderer, FlutterStreamHandler { + + private var eventSink: FlutterEventSink? + + private var channel: FlutterEventChannel? + + public func onListen(withArguments arguments: Any?, eventSink events: @escaping FlutterEventSink) -> FlutterError? { + self.eventSink = events + return nil + } + + public func onCancel(withArguments arguments: Any?) -> FlutterError? { + eventSink = nil + return nil + } + + public let isCentered: Bool + public let smoothingFactor: Float + + public var bands: [Float] + + private let _processor: AudioVisualizeProcessor + private weak var _track: AudioTrack? + + public init(track: AudioTrack?, + binaryMessenger: FlutterBinaryMessenger, + bandCount: Int = 7, + isCentered: Bool = true, + smoothingFactor: Float = 0.3) + { + self.isCentered = isCentered + self.smoothingFactor = smoothingFactor + bands = Array(repeating: 0.0, count: bandCount) + _processor = AudioVisualizeProcessor(bandsCount: bandCount) + _track = track + super.init() + _track?.add(audioRenderer: self) + let channelName = "io.livekit.audio.visualizer/eventChannel-" + (track?.mediaTrack.trackId ?? "") + channel = FlutterEventChannel(name: channelName, binaryMessenger: binaryMessenger) + channel?.setStreamHandler(self) + } + + deinit { + _track?.remove(audioRenderer: self) + channel?.setStreamHandler(nil) + } + + public func render(pcmBuffer: AVAudioPCMBuffer) { + let newBands = _processor.process(pcmBuffer: pcmBuffer) + guard var newBands else { return } + + // If centering is enabled, rearrange the normalized bands + if isCentered { + newBands.sort(by: >) + newBands = centerBands(newBands) + } + + DispatchQueue.main.async { [weak self] in + guard let self else { return } + + self.bands = zip(self.bands, newBands).map { old, new in + self._smoothTransition(from: old, to: new, factor: self.smoothingFactor) + } + eventSink?(self.bands) + } + } + + // MARK: - Private + + /// Centers the sorted bands by placing higher values in the middle. + @inline(__always) private func centerBands(_ sortedBands: [Float]) -> [Float] { + var centeredBands = [Float](repeating: 0, count: sortedBands.count) + var leftIndex = sortedBands.count / 2 + var rightIndex = leftIndex + + for (index, value) in sortedBands.enumerated() { + if index % 2 == 0 { + // Place value to the right + centeredBands[rightIndex] = value + rightIndex += 1 + } else { + // Place value to the left + leftIndex -= 1 + centeredBands[leftIndex] = value + } + } + + return centeredBands + } + + /// Applies an easing function to smooth the transition. + @inline(__always) private func _smoothTransition(from oldValue: Float, to newValue: Float, factor: Float) -> Float { + // Calculate the delta change between the old and new value + let delta = newValue - oldValue + // Apply an ease-in-out cubic easing curve + let easedFactor = _easeInOutCubic(t: factor) + // Calculate and return the smoothed value + return oldValue + delta * easedFactor + } + + /// Easing function: ease-in-out cubic + @inline(__always) private func _easeInOutCubic(t: Float) -> Float { + t < 0.5 ? 4 * t * t * t : 1 - pow(-2 * t + 2, 3) / 2 + } +}