From da21284f2a1dd173014162734b36cbb1093bf709 Mon Sep 17 00:00:00 2001 From: CloudWebRTC Date: Mon, 22 Jul 2024 18:21:04 +0800 Subject: [PATCH] feat: support rendering callback for rtc video source. (#43) * feat: support rendering callback for rtc video source. * update. * bump version. * update. * texture dispose. --- Runtime/Scripts/CameraVideoSource.cs | 9 +++--- Runtime/Scripts/RtcVideoSource.cs | 43 ++++++++++++++++++++++++--- Runtime/Scripts/ScreenVideoSource.cs | 7 +++-- Runtime/Scripts/TextureVideoSource.cs | 12 ++++++-- Runtime/Scripts/VideoStream.cs | 4 +-- package.json | 2 +- 6 files changed, 62 insertions(+), 15 deletions(-) diff --git a/Runtime/Scripts/CameraVideoSource.cs b/Runtime/Scripts/CameraVideoSource.cs index bbeb994..df7512d 100644 --- a/Runtime/Scripts/CameraVideoSource.cs +++ b/Runtime/Scripts/CameraVideoSource.cs @@ -52,12 +52,12 @@ private void ClearRenderTexture() } // Read the texture data into a native array asynchronously - protected override void ReadBuffer() + protected override bool ReadBuffer() { if (_reading) - return; + return false; _reading = true; - + var textureChanged = false; try { if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight()) @@ -69,6 +69,7 @@ protected override void ReadBuffer() _dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat); Camera.targetTexture = _dest as RenderTexture; _data = new NativeArray(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent); + textureChanged = true; } ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture); AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback); @@ -77,7 +78,7 @@ protected override void ReadBuffer() { Utils.Error(e); } - + return textureChanged; } } } diff --git a/Runtime/Scripts/RtcVideoSource.cs b/Runtime/Scripts/RtcVideoSource.cs index a44b894..17c7673 100644 --- a/Runtime/Scripts/RtcVideoSource.cs +++ b/Runtime/Scripts/RtcVideoSource.cs @@ -26,6 +26,10 @@ public enum VideoStreamSource public abstract int GetWidth(); public abstract int GetHeight(); + public delegate void TextureReceiveDelegate(Texture2D tex2d); + /// Called when we receive a new texture (first texture or the resolution changed) + public event TextureReceiveDelegate TextureReceived; + protected Texture _dest; protected NativeArray _data; protected VideoStreamSource _sourceType; @@ -35,6 +39,7 @@ public enum VideoStreamSource protected bool _requestPending = false; protected bool isDisposed = true; protected bool _playing = false; + private Texture2D _texture2D = null; internal RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType) { @@ -121,12 +126,41 @@ public virtual void Stop() _playing = false; } + private void LoadToTexture2D(Texture2D tex, RenderTexture rTex) + { + var old_rt = RenderTexture.active; + RenderTexture.active = rTex; + + tex.ReadPixels(new Rect(0, 0, rTex.width, rTex.height), 0, 0); + tex.Apply(); + + RenderTexture.active = old_rt; + } + public IEnumerator Update() { while (_playing) { yield return null; - ReadBuffer(); + var textureChanged = ReadBuffer(); + + if(textureChanged) + { + if (_texture2D == null) + { + _texture2D = new Texture2D(_dest.width, _dest.height, TextureFormat.RGB24, false); + } else + { + _texture2D.Reinitialize(_dest.width, _dest.height); + } + TextureReceived?.Invoke(_texture2D); + } + + if(TextureReceived.GetInvocationList().Length > 0) + { + LoadToTexture2D(_texture2D, _dest as RenderTexture); + } + SendFrame(); } @@ -136,13 +170,14 @@ public IEnumerator Update() public virtual void Dispose() { if (!isDisposed) - { - _data.Dispose(); + { + if (_data != null) _data.Dispose(); + if (_texture2D != null) UnityEngine.Object.Destroy(_texture2D); isDisposed = true; } } - protected abstract void ReadBuffer(); + protected abstract bool ReadBuffer(); protected virtual bool SendFrame() { diff --git a/Runtime/Scripts/ScreenVideoSource.cs b/Runtime/Scripts/ScreenVideoSource.cs index c23cade..c6204fe 100644 --- a/Runtime/Scripts/ScreenVideoSource.cs +++ b/Runtime/Scripts/ScreenVideoSource.cs @@ -49,11 +49,12 @@ private void ClearRenderTexture() } // Read the texture data into a native array asynchronously - protected override void ReadBuffer() + protected override bool ReadBuffer() { if (_reading) - return; + return false; _reading = true; + var textureChanged = false; try { if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight()) @@ -64,6 +65,7 @@ protected override void ReadBuffer() _bufferType = GetVideoBufferType(_textureFormat); _dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat); _data = new NativeArray(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent); + textureChanged = true; } ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture); AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback); @@ -72,6 +74,7 @@ protected override void ReadBuffer() { Utils.Error(e); } + return textureChanged; } protected override bool SendFrame() diff --git a/Runtime/Scripts/TextureVideoSource.cs b/Runtime/Scripts/TextureVideoSource.cs index 5d9f6dc..09eaa92 100644 --- a/Runtime/Scripts/TextureVideoSource.cs +++ b/Runtime/Scripts/TextureVideoSource.cs @@ -34,11 +34,12 @@ public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBuf } // Read the texture data into a native array asynchronously - protected override void ReadBuffer() + protected override bool ReadBuffer() { if (_reading) - return; + return false; _reading = true; + var textureChanged = false; if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels)) { if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight()) @@ -48,17 +49,24 @@ protected override void ReadBuffer() _bufferType = GetVideoBufferType(_textureFormat); _data = new NativeArray(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent); _dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false); + textureChanged = true; } Graphics.CopyTexture(Texture, _dest); } else { + if(_dest == null || _dest != Texture) + { + textureChanged = true; + } + _dest = Texture; _textureFormat = GraphicsFormatUtility.GetTextureFormat(Texture.graphicsFormat); _bufferType = GetVideoBufferType(_textureFormat); } AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback); + return textureChanged; } } } diff --git a/Runtime/Scripts/VideoStream.cs b/Runtime/Scripts/VideoStream.cs index 9379b80..0ab4a0a 100644 --- a/Runtime/Scripts/VideoStream.cs +++ b/Runtime/Scripts/VideoStream.cs @@ -71,7 +71,7 @@ private void Dispose(bool disposing) { if (disposing) VideoBuffer?.Dispose(); - + if (Texture != null) UnityEngine.Object.Destroy(Texture); _disposed = true; } } @@ -106,7 +106,7 @@ public IEnumerator Update() var textureChanged = false; if (Texture == null || Texture.width != rWidth || Texture.height != rHeight) { - if (Texture != null) UnityEngine.Object.Destroy(Texture); + if (Texture != null) UnityEngine.Object.Destroy(Texture); Texture = new Texture2D((int)rWidth, (int)rHeight, TextureFormat.RGBA32, false); Texture.ignoreMipmapLimit = false; textureChanged = true; diff --git a/package.json b/package.json index 0f9fd99..7cf5d73 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "io.livekit.livekit-sdk", - "version": "1.0.0", + "version": "1.0.1", "displayName": "LiveKit SDK", "description": "LiveKit", "unity": "2021.3",