diff --git a/README.md b/README.md index 3596d71..3af68e8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ - [x] Windows - [x] MacOS - [x] Linux -- [ ] iOS +- [x] iOS - [x] Android - [ ] WebGL @@ -44,7 +44,11 @@ select `Unity-iPhone` -> `TARGETS` -> `UnityFramework` -> `General` -> `Framewor add the following frameworks: -`OpenGLES.framework` `MetalKit.framework` +`OpenGLES.framework` `MetalKit.framework` `GLKit.framework` `MetalKit.framework` `VideoToolBox.framework` `Network.framework` + +add other linker flags: + +`-ObjC` Since `libiPhone-lib.a` has built-in old versions of `celt` and `libvpx` (This will cause the opus and vp8/vp9 codecs to not be called correctly and cause a crash.), so you need to adjust the link order to ensure that it is linked to `liblivekit_ffi.a` first. @@ -52,7 +56,8 @@ The fix is ​​to remove and re-add `libiPhone-lib.a` from `Frameworks and Lib ## Examples -You can find examples in the [unity-example](https://github.com/livekit-examples/unity-example.git). +And you can also follow the example app in the [unity-example](https://github.com/livekit-examples/unity-example.git) to see how to use the SDK. + ### Connect to a room @@ -77,15 +82,18 @@ IEnumerator Start() ```cs // Publish Microphone -var source = GetComponent(); -source.clip = Microphone.Start("MacBook Pro Microphone", true, 2, 48000); +var localSid = "my-audio-source"; +GameObject audObject = new GameObject(localSid); +var source = audObject.AddComponent(); +source.clip = Microphone.Start(Microphone.devices[0], true, 2, (int)RtcAudioSource.DefaultSampleRate); source.loop = true; -Ssurce.Play(); -var rtcSource = new RtcAudioSource(Source); -var track = LocalAudioTrack.CreateAudioTrack("my-track", rtcSource); +var rtcSource = new RtcAudioSource(source); +var track = LocalAudioTrack.CreateAudioTrack("my-audio-track", rtcSource, room); var options = new TrackPublishOptions(); +options.AudioEncoding = new AudioEncoding(); +options.AudioEncoding.MaxBitrate = 64000; options.Source = TrackSource.SourceMicrophone; var publish = room.LocalParticipant.PublishTrack(track, options); @@ -95,6 +103,8 @@ if (!publish.IsError) { Debug.Log("Track published!"); } + +rtcSource.Start(); ``` ### Publishing a texture (e.g Unity Camera) @@ -105,19 +115,27 @@ rt.Create(); Camera.main.targetTexture = rt; var source = new TextureVideoSource(rt); -var track = LocalVideoTrack.CreateVideoTrack("my-track", source); +var track = LocalVideoTrack.CreateVideoTrack("my-video-track", source, room); var options = new TrackPublishOptions(); -options.VideoCodec = VideoCodec.H264; +options.VideoCodec = VideoCodec.Vp8; +var videoCoding = new VideoEncoding(); +videoCoding.MaxBitrate = 512000; +videoCoding.MaxFramerate = frameRate; +options.VideoEncoding = videoCoding; +options.Simulcast = true; options.Source = TrackSource.SourceCamera; -var publish = _room.LocalParticipant.PublishTrack(track, options); +var publish = room.LocalParticipant.PublishTrack(track, options); yield return publish; if (!publish.IsError) { Debug.Log("Track published!"); } + +source.Start(); +StartCoroutine(source.Update()); ``` ### Receiving tracks @@ -149,7 +167,8 @@ void TrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, Rem } else if (track is RemoteAudioTrack audioTrack) { - var source = GetComponent(); + GameObject audObject = new GameObject(audioTrack.Sid); + var source = audObject.AddComponent(); var stream = new AudioStream(audioTrack, source); // Audio is being played on the source .. } diff --git a/Runtime/Plugins/ffi-ios-arm64/liblivekit_ffi.a.meta b/Runtime/Plugins/ffi-ios-arm64/liblivekit_ffi.a.meta index 6f05164..1be97a9 100644 --- a/Runtime/Plugins/ffi-ios-arm64/liblivekit_ffi.a.meta +++ b/Runtime/Plugins/ffi-ios-arm64/liblivekit_ffi.a.meta @@ -76,7 +76,7 @@ PluginImporter: AddToEmbeddedBinaries: false CPU: ARM64 CompileFlags: - FrameworkDependencies: OpenGLES;MetalKit; + FrameworkDependencies: OpenGLES;GLKit;MetalKit;Network;VideoToolbox; userData: assetBundleName: assetBundleVariant: diff --git a/Runtime/Scripts/AudioSource.cs b/Runtime/Scripts/AudioSource.cs index 976239a..1fb2f90 100644 --- a/Runtime/Scripts/AudioSource.cs +++ b/Runtime/Scripts/AudioSource.cs @@ -10,7 +10,11 @@ namespace LiveKit { public class RtcAudioSource { +#if UNITY_IOS + public static uint DefaultSampleRate = 24000; +#else public static uint DefaultSampleRate = 48000; +#endif public static uint DefaultChannels = 2; private AudioSource _audioSource; @@ -38,7 +42,6 @@ public RtcAudioSource(AudioSource source) using var response = request.Send(); FfiResponse res = response; _info = res.NewAudioSource.Source.Info; - //TODO pooling handles Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle); UpdateSource(source); } diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs index 0cb968d..5ce2666 100644 --- a/Runtime/Scripts/Internal/FFIClient.cs +++ b/Runtime/Scripts/Internal/FFIClient.cs @@ -170,17 +170,15 @@ public FfiResponse SendRequest(FfiRequest request) using var memory = memoryPool.Memory(request); var data = memory.Span(); request.WriteTo(data); - fixed (byte* requestDataPtr = data) { var handle = NativeMethods.FfiNewRequest( requestDataPtr, data.Length, out byte* dataPtr, - out int dataLen + out UIntPtr dataLen ); - - var dataSpan = new Span(dataPtr, dataLen); + var dataSpan = new Span(dataPtr, (int)dataLen.ToUInt64()); var response = responseParser.ParseFrom(dataSpan)!; NativeMethods.FfiDropHandle(handle); return response; @@ -198,7 +196,7 @@ out int dataLen [AOT.MonoPInvokeCallback(typeof(FFICallbackDelegate))] - static unsafe void FFICallback(IntPtr data, int size) + static unsafe void FFICallback(UIntPtr data, UIntPtr size) { #if NO_LIVEKIT_MODE return; @@ -206,7 +204,7 @@ static unsafe void FFICallback(IntPtr data, int size) if (_isDisposed) return; - var respData = new Span(data.ToPointer()!, size); + var respData = new Span(data.ToPointer()!, (int)size.ToUInt64()); var response = FfiEvent.Parser!.ParseFrom(respData); // Run on the main thread, the order of execution is guaranteed by Unity @@ -239,9 +237,8 @@ static unsafe void FFICallback(IntPtr data, int size) case FfiEvent.MessageOneofCase.Disconnect: Instance.DisconnectReceived?.Invoke(r.Disconnect!); break; - /*case FfiEvent.MessageOneofCase. ParticipantEvent: - Instance.ParticipantEventReceived?.Invoke(response.ParticipantEvent); - break;*/ + case FfiEvent.MessageOneofCase.PublishTranscription: + break; case FfiEvent.MessageOneofCase.VideoStreamEvent: Instance.VideoStreamEventReceived?.Invoke(r.VideoStreamEvent!); break; @@ -250,6 +247,7 @@ static unsafe void FFICallback(IntPtr data, int size) break; case FfiEvent.MessageOneofCase.CaptureAudioFrame: break; + case FfiEvent.MessageOneofCase.GetStats: case FfiEvent.MessageOneofCase.Panic: break; default: diff --git a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs index ac5e978..de4ca1c 100644 --- a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs +++ b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs @@ -5,7 +5,8 @@ namespace LiveKit.Internal { [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - internal delegate void FFICallbackDelegate(IntPtr data, int size); + + internal delegate void FFICallbackDelegate(UIntPtr data, UIntPtr size); // Callbacks internal delegate void PublishTrackDelegate(PublishTrackCallback e); diff --git a/Runtime/Scripts/Internal/NativeMethods.cs b/Runtime/Scripts/Internal/NativeMethods.cs index d2e58df..a03a890 100644 --- a/Runtime/Scripts/Internal/NativeMethods.cs +++ b/Runtime/Scripts/Internal/NativeMethods.cs @@ -3,6 +3,8 @@ using System.Runtime.ConstrainedExecution; using System.Runtime.InteropServices; +using FfiHandleId = System.IntPtr; + namespace LiveKit.Internal { [SuppressUnmanagedCodeSecurity] @@ -16,13 +18,12 @@ internal static class NativeMethods [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_drop_handle")] [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)] - internal extern static bool FfiDropHandle(IntPtr handleId); + internal extern static bool FfiDropHandle(FfiHandleId handleId); [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_request")] - internal extern static unsafe IntPtr FfiNewRequest(byte* data, int len, out byte* dataPtr, out int dataLen); + internal extern static unsafe FfiHandleId FfiNewRequest(byte* data, int len, out byte* dataPtr, out UIntPtr dataLen); - //TODO optimise FfiHandle, can be replaced by FfiHandleId = uint64_t [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize")] - internal extern static IntPtr LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs); + internal extern static FfiHandleId LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs); } } \ No newline at end of file diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs index f468463..7a3b019 100644 --- a/Runtime/Scripts/RtcVideoSource.cs +++ b/Runtime/Scripts/RtcVideoSource.cs @@ -41,6 +41,9 @@ public RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType) _bufferType = bufferType; using var request = FFIBridge.Instance.NewRequest(); var newVideoSource = request.request; + newVideoSource.Resolution = request.TempResource(); + newVideoSource.Resolution.Width = 1280; + newVideoSource.Resolution.Height = 720; newVideoSource.Type = VideoSourceType.VideoSourceNative; using var response = request.Send(); FfiResponse res = response;