feat: Support for iOS. (#36)

* feat: Support for iOS. * changes. * update ffi protocol. * Update README.md * update Proto. * Add VideoToolBox to FrameworkDependencies. * update README.md. * update README.md.
livekit · Jul 2, 2024 · abd5d81 · abd5d81
1 parent 1d8b514
commit abd5d81
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 28 deletions.
diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@
 - [x] Windows
 - [x] MacOS
 - [x] Linux
-- [ ] iOS
+- [x] iOS
 - [x] Android
 - [ ] WebGL
 
@@ -44,15 +44,20 @@ select `Unity-iPhone` -> `TARGETS` -> `UnityFramework` -> `General` -> `Framewor
 
 add the following frameworks:
 
-`OpenGLES.framework` `MetalKit.framework`
+`OpenGLES.framework` `MetalKit.framework` `GLKit.framework` `MetalKit.framework` `VideoToolBox.framework` `Network.framework`
+
+add other linker flags:
+
+`-ObjC`
 
 Since `libiPhone-lib.a` has built-in old versions of `celt` and `libvpx` (This will cause the opus and vp8/vp9 codecs to not be called correctly and cause a crash.), so you need to adjust the link order to ensure that it is linked to `liblivekit_ffi.a` first.
 
 The fix is to remove and re-add `libiPhone-lib.a` from `Frameworks and Libraries`, making sure to link after `liblivekit_ffi.a`.
 
 ## Examples
 
-You can find examples in the [unity-example](https://github.com/livekit-examples/unity-example.git).
+And you can also follow the example app in the [unity-example](https://github.com/livekit-examples/unity-example.git) to see how to use the SDK.
+
 
 ### Connect to a room
 
@@ -77,15 +82,18 @@ IEnumerator Start()
 
 ```cs
 // Publish Microphone
-var source = GetComponent<AudioSource>();
-source.clip = Microphone.Start("MacBook Pro Microphone", true, 2, 48000);
+var localSid = "my-audio-source";
+GameObject audObject = new GameObject(localSid);
+var source = audObject.AddComponent<AudioSource>();
+source.clip = Microphone.Start(Microphone.devices[0], true, 2, (int)RtcAudioSource.DefaultSampleRate);
 source.loop = true;
-Ssurce.Play();
 
-var rtcSource = new RtcAudioSource(Source);
-var track = LocalAudioTrack.CreateAudioTrack("my-track", rtcSource);
+var rtcSource = new RtcAudioSource(source);
+var track = LocalAudioTrack.CreateAudioTrack("my-audio-track", rtcSource, room);
 
 var options = new TrackPublishOptions();
+options.AudioEncoding = new AudioEncoding();
+options.AudioEncoding.MaxBitrate = 64000;
 options.Source = TrackSource.SourceMicrophone;
 
 var publish = room.LocalParticipant.PublishTrack(track, options);
@@ -95,6 +103,8 @@ if (!publish.IsError)
 {
     Debug.Log("Track published!");
 }
+
+rtcSource.Start();
 ```
 
 ### Publishing a texture (e.g Unity Camera)
@@ -105,19 +115,27 @@ rt.Create();
 Camera.main.targetTexture = rt;
 
 var source = new TextureVideoSource(rt);
-var track = LocalVideoTrack.CreateVideoTrack("my-track", source);
+var track = LocalVideoTrack.CreateVideoTrack("my-video-track", source, room);
 
 var options = new TrackPublishOptions();
-options.VideoCodec = VideoCodec.H264;
+options.VideoCodec = VideoCodec.Vp8;
+var videoCoding = new VideoEncoding();
+videoCoding.MaxBitrate = 512000;
+videoCoding.MaxFramerate = frameRate;
+options.VideoEncoding = videoCoding;
+options.Simulcast = true;
 options.Source = TrackSource.SourceCamera;
 
-var publish = _room.LocalParticipant.PublishTrack(track, options);
+var publish = room.LocalParticipant.PublishTrack(track, options);
 yield return publish;
 
 if (!publish.IsError)
 {
     Debug.Log("Track published!");
 }
+
+source.Start();
+StartCoroutine(source.Update());
 ```
 
 ### Receiving tracks
@@ -149,7 +167,8 @@ void TrackSubscribed(IRemoteTrack track, RemoteTrackPublication publication, Rem
     }
     else if (track is RemoteAudioTrack audioTrack)
     {
-        var source = GetComponent<AudioSource>();
+        GameObject audObject = new GameObject(audioTrack.Sid);
+        var source = audObject.AddComponent<AudioSource>();
         var stream = new AudioStream(audioTrack, source);
         // Audio is being played on the source ..
     }

diff --git a/Runtime/Plugins/ffi-ios-arm64/liblivekit_ffi.a.meta b/Runtime/Plugins/ffi-ios-arm64/liblivekit_ffi.a.meta
diff --git a/Runtime/Scripts/AudioSource.cs b/Runtime/Scripts/AudioSource.cs
@@ -10,7 +10,11 @@ namespace LiveKit
 {
     public class RtcAudioSource
     {
+#if UNITY_IOS
+        public static uint DefaultSampleRate = 24000;
+#else
         public static uint DefaultSampleRate = 48000;
+#endif
         public static uint DefaultChannels = 2;
 
         private AudioSource _audioSource;
@@ -38,7 +42,6 @@ public RtcAudioSource(AudioSource source)
             using var response = request.Send();
             FfiResponse res = response;
             _info = res.NewAudioSource.Source.Info;
-            //TODO pooling handles
             Handle = FfiHandle.FromOwnedHandle(res.NewAudioSource.Source.Handle);
             UpdateSource(source);
         }

diff --git a/Runtime/Scripts/Internal/FFIClient.cs b/Runtime/Scripts/Internal/FFIClient.cs
@@ -170,17 +170,15 @@ public FfiResponse SendRequest(FfiRequest request)
                     using var memory = memoryPool.Memory(request);
                     var data = memory.Span();
                     request.WriteTo(data);
-
                     fixed (byte* requestDataPtr = data)
                     {
                         var handle = NativeMethods.FfiNewRequest(
                             requestDataPtr,
                             data.Length,
                             out byte* dataPtr,
-                            out int dataLen
+                            out UIntPtr dataLen
                         );
-
-                        var dataSpan = new Span<byte>(dataPtr, dataLen);
+                        var dataSpan = new Span<byte>(dataPtr, (int)dataLen.ToUInt64());
                         var response = responseParser.ParseFrom(dataSpan)!;
                         NativeMethods.FfiDropHandle(handle);
                         return response;
@@ -198,15 +196,15 @@ out int dataLen
 
 
         [AOT.MonoPInvokeCallback(typeof(FFICallbackDelegate))]
-        static unsafe void FFICallback(IntPtr data, int size)
+        static unsafe void FFICallback(UIntPtr data, UIntPtr size)
         { 
 #if NO_LIVEKIT_MODE
             return;
 #endif
 
             if (_isDisposed) return;
 
-            var respData = new Span<byte>(data.ToPointer()!, size);
+            var respData = new Span<byte>(data.ToPointer()!, (int)size.ToUInt64());
             var response = FfiEvent.Parser!.ParseFrom(respData);
 
             // Run on the main thread, the order of execution is guaranteed by Unity
@@ -239,9 +237,8 @@ static unsafe void FFICallback(IntPtr data, int size)
                     case FfiEvent.MessageOneofCase.Disconnect:
                         Instance.DisconnectReceived?.Invoke(r.Disconnect!);
                         break;
-                    /*case FfiEvent.MessageOneofCase. ParticipantEvent:
-                            Instance.ParticipantEventReceived?.Invoke(response.ParticipantEvent);
-                            break;*/
+                    case FfiEvent.MessageOneofCase.PublishTranscription:
+                        break;
                     case FfiEvent.MessageOneofCase.VideoStreamEvent:
                         Instance.VideoStreamEventReceived?.Invoke(r.VideoStreamEvent!);
                         break;
@@ -250,6 +247,7 @@ static unsafe void FFICallback(IntPtr data, int size)
                         break;
                     case FfiEvent.MessageOneofCase.CaptureAudioFrame:
                         break;
+                    case FfiEvent.MessageOneofCase.GetStats:
                     case FfiEvent.MessageOneofCase.Panic:
                         break;
                     default:

diff --git a/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs b/Runtime/Scripts/Internal/FFIClients/FFIEvents.cs
@@ -5,7 +5,8 @@
 namespace LiveKit.Internal
 {
     [UnmanagedFunctionPointer(CallingConvention.Cdecl)]
-    internal delegate void FFICallbackDelegate(IntPtr data, int size);
+
+    internal delegate void FFICallbackDelegate(UIntPtr data, UIntPtr size);
 
     // Callbacks
     internal delegate void PublishTrackDelegate(PublishTrackCallback e);

diff --git a/Runtime/Scripts/Internal/NativeMethods.cs b/Runtime/Scripts/Internal/NativeMethods.cs
@@ -3,6 +3,8 @@
 using System.Runtime.ConstrainedExecution;
 using System.Runtime.InteropServices;
 
+using FfiHandleId = System.IntPtr;
+
 namespace LiveKit.Internal
 {
     [SuppressUnmanagedCodeSecurity]
@@ -16,13 +18,12 @@ internal static class NativeMethods
 
         [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_drop_handle")]
         [ReliabilityContract(Consistency.WillNotCorruptState, Cer.MayFail)]
-        internal extern static bool FfiDropHandle(IntPtr handleId);
+        internal extern static bool FfiDropHandle(FfiHandleId handleId);
 
         [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_request")]
-        internal extern static unsafe IntPtr FfiNewRequest(byte* data, int len, out byte* dataPtr, out int dataLen);
+        internal extern static unsafe FfiHandleId FfiNewRequest(byte* data, int len, out byte* dataPtr, out UIntPtr dataLen);
 
-        //TODO optimise FfiHandle, can be replaced by FfiHandleId = uint64_t
         [DllImport(Lib, CallingConvention = CallingConvention.Cdecl, EntryPoint = "livekit_ffi_initialize")]
-        internal extern static IntPtr LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs);
+        internal extern static FfiHandleId LiveKitInitialize(FFICallbackDelegate cb, bool captureLogs);
     }
 }
diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs
@@ -41,6 +41,9 @@ public RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType)
             _bufferType = bufferType;
             using var request = FFIBridge.Instance.NewRequest<NewVideoSourceRequest>();
             var newVideoSource = request.request;
+            newVideoSource.Resolution = request.TempResource<VideoSourceResolution>();
+            newVideoSource.Resolution.Width = 1280;
+            newVideoSource.Resolution.Height = 720;
             newVideoSource.Type = VideoSourceType.VideoSourceNative;
             using var response = request.Send();
             FfiResponse res = response;