feat: support rendering callback for rtc video source. (#43)

* feat: support rendering callback for rtc video source. * update. * bump version. * update. * texture dispose.
livekit · Jul 22, 2024 · da21284 · da21284
1 parent dc1d8c7
commit da21284
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 15 deletions.
diff --git a/Runtime/Scripts/CameraVideoSource.cs b/Runtime/Scripts/CameraVideoSource.cs
@@ -52,12 +52,12 @@ private void ClearRenderTexture()
         }
 
         // Read the texture data into a native array asynchronously
-        protected override void ReadBuffer()
+        protected override bool ReadBuffer()
         {
             if (_reading)
-                return;
+                return false;
             _reading = true;
-
+            var textureChanged = false;
             try
             {
                 if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
@@ -69,6 +69,7 @@ protected override void ReadBuffer()
                     _dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
                     Camera.targetTexture = _dest as RenderTexture;
                     _data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
+                    textureChanged = true;
                 }
                 ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
                 AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
@@ -77,7 +78,7 @@ protected override void ReadBuffer()
             {
                 Utils.Error(e);
             }
-
+            return textureChanged;
         }
     }
 }

diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs
@@ -26,6 +26,10 @@ public enum VideoStreamSource
         public abstract int GetWidth();
         public abstract int GetHeight();
 
+        public delegate void TextureReceiveDelegate(Texture2D tex2d);
+        /// Called when we receive a new texture (first texture or the resolution changed)
+        public event TextureReceiveDelegate TextureReceived;
+
         protected Texture _dest;
         protected NativeArray<byte> _data;
         protected VideoStreamSource _sourceType;
@@ -35,6 +39,7 @@ public enum VideoStreamSource
         protected bool _requestPending = false;
         protected bool isDisposed = true;
         protected bool _playing = false;
+        private Texture2D _texture2D = null;
 
         internal RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType)
         {
@@ -121,12 +126,41 @@ public virtual void Stop()
             _playing = false; 
         }
 
+        private void LoadToTexture2D(Texture2D tex, RenderTexture rTex)
+        {
+            var old_rt = RenderTexture.active;
+            RenderTexture.active = rTex;
+
+            tex.ReadPixels(new Rect(0, 0, rTex.width, rTex.height), 0, 0);
+            tex.Apply();
+
+            RenderTexture.active = old_rt;
+        }
+
         public IEnumerator Update()
         {
             while (_playing)
             {
                 yield return null;
-                ReadBuffer();
+                var textureChanged = ReadBuffer();
+
+                if(textureChanged)
+                {
+                    if (_texture2D == null)
+                    {
+                        _texture2D = new Texture2D(_dest.width, _dest.height, TextureFormat.RGB24, false);
+                    } else
+                    {
+                        _texture2D.Reinitialize(_dest.width, _dest.height);
+                    }
+                    TextureReceived?.Invoke(_texture2D);
+                }
+
+                if(TextureReceived.GetInvocationList().Length > 0)
+                {
+                    LoadToTexture2D(_texture2D, _dest as RenderTexture);
+                }
+
                 SendFrame();
             }
 
@@ -136,13 +170,14 @@ public IEnumerator Update()
         public virtual void Dispose()
         {
             if (!isDisposed)
-            { 
-                _data.Dispose();
+            {
+                if (_data != null) _data.Dispose();
+                if (_texture2D != null) UnityEngine.Object.Destroy(_texture2D);
                 isDisposed = true;
             }
         }
 
-        protected abstract void ReadBuffer();
+        protected abstract bool ReadBuffer();
 
         protected virtual bool SendFrame()
         {

diff --git a/Runtime/Scripts/ScreenVideoSource.cs b/Runtime/Scripts/ScreenVideoSource.cs
@@ -49,11 +49,12 @@ private void ClearRenderTexture()
         }
 
         // Read the texture data into a native array asynchronously
-        protected override void ReadBuffer()
+        protected override bool ReadBuffer()
         {
             if (_reading)
-                return;
+                return false;
             _reading = true;
+            var textureChanged = false;
             try
             {
                 if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
@@ -64,6 +65,7 @@ protected override void ReadBuffer()
                     _bufferType = GetVideoBufferType(_textureFormat);
                     _dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
                     _data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
+                    textureChanged = true;
                 }
                 ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
                 AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
@@ -72,6 +74,7 @@ protected override void ReadBuffer()
             {
                 Utils.Error(e);
             }
+            return textureChanged;
         }
 
         protected override bool SendFrame()

diff --git a/Runtime/Scripts/TextureVideoSource.cs b/Runtime/Scripts/TextureVideoSource.cs
@@ -34,11 +34,12 @@ public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBuf
         }
 
         // Read the texture data into a native array asynchronously
-        protected override void ReadBuffer()
+        protected override bool ReadBuffer()
         {
             if (_reading)
-                return;
+                return false;
             _reading = true;
+            var textureChanged = false;
             if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels))
             {
                 if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
@@ -48,17 +49,24 @@ protected override void ReadBuffer()
                     _bufferType = GetVideoBufferType(_textureFormat);
                     _data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
                     _dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
+                    textureChanged = true;
                 }
                 Graphics.CopyTexture(Texture, _dest);
             }
             else
             {
+                if(_dest == null || _dest != Texture)
+                {
+                    textureChanged = true;
+                }
+
                 _dest = Texture;
                 _textureFormat = GraphicsFormatUtility.GetTextureFormat(Texture.graphicsFormat);
                 _bufferType = GetVideoBufferType(_textureFormat);
             }
 
             AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
+            return textureChanged;
         }
     }
 }

diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs
@@ -71,7 +71,7 @@ private void Dispose(bool disposing)
             {
                 if (disposing)
                     VideoBuffer?.Dispose();
-
+                if (Texture != null) UnityEngine.Object.Destroy(Texture);
                 _disposed = true;
             }
         }
@@ -106,7 +106,7 @@ public IEnumerator Update()
                 var textureChanged = false;
                 if (Texture == null || Texture.width != rWidth || Texture.height != rHeight)
                 {
-                   if (Texture != null) UnityEngine.Object.Destroy(Texture);
+                    if (Texture != null) UnityEngine.Object.Destroy(Texture);
                     Texture = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false);
                     Texture.ignoreMipmapLimit = false;
                     textureChanged = true;

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "io.livekit.livekit-sdk",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "displayName": "LiveKit SDK",
   "description": "LiveKit",
   "unity": "2021.3",