Skip to content

Commit

Permalink
fix bug for RtcVideoSources. (#42)
Browse files Browse the repository at this point in the history 
* fix bug in ScreenVideoSource.

* fix bug for CameraVideoSource.

* Update README.md
  • Loading branch information
@cloudwebrtc
cloudwebrtc authored Jul 3, 2024
1 parent 87b8f0e commit 175aef8
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 36 deletions.
11 changes: 10 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,19 @@ rtcSource.Start();
### Publishing a texture (e.g Unity Camera)

```cs
// publish a WebCamera
//var source = new TextureVideoSource(webCamTexture);
// Publish the entire screen
//var source = new ScreenVideoSource();
// Publishing a Unity Camera
//Camera.main.enabled = true;
//var source = new CameraVideoSource(Camera.main);
var rt = new UnityEngine.RenderTexture(1920, 1080, 24, RenderTextureFormat.ARGB32);
rt.Create();
Camera.main.targetTexture = rt;

var source = new TextureVideoSource(rt);
var track = LocalVideoTrack.CreateVideoTrack("my-video-track", source, room);

Expand Down
32 changes: 26 additions & 6 deletions Runtime/Scripts/CameraVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@
using LiveKit.Proto;
using LiveKit.Internal;
using UnityEngine.Rendering;
using UnityEngine.Experimental.Rendering;
using System;
using Unity.Collections;

namespace LiveKit
{
public class CameraVideoSource : RtcVideoSource
{
private TextureFormat _textureFormat;

public Camera Camera { get; }

public override int GetWidth()
Expand All @@ -23,11 +27,7 @@ public override int GetHeight()
public CameraVideoSource(Camera camera, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType)
{
Camera = camera;

var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, targetFormat);
camera.targetTexture = _dest as RenderTexture;
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(bufferType), Allocator.Persistent);
base.Init();
}

~CameraVideoSource()
Expand Down Expand Up @@ -57,7 +57,27 @@ protected override void ReadBuffer()
if (_reading)
return;
_reading = true;
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, GetTextureFormat(_bufferType), OnReadback);

try
{
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
{
var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
var compatibleFormat = SystemInfo.GetCompatibleFormat(targetFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
Camera.targetTexture = _dest as RenderTexture;
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
}
ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
}
catch (Exception e)
{
Utils.Error(e);
}

}
}
}
Expand Down
58 changes: 46 additions & 12 deletions Runtime/Scripts/RtcVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using Unity.Collections.LowLevel.Unsafe;
using LiveKit.Internal.FFIClients.Requests;
using System.Collections;
using UnityEngine.Experimental.Rendering;

namespace LiveKit
{
Expand All @@ -19,7 +20,8 @@ public enum VideoStreamSource
Camera = 2
}

internal FfiHandle Handle { get; }

internal FfiHandle Handle { get; set; }

public abstract int GetWidth();
public abstract int GetHeight();
Expand All @@ -34,21 +36,29 @@ public enum VideoStreamSource
protected bool isDisposed = true;
protected bool _playing = false;

public RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType)
internal RtcVideoSource(VideoStreamSource sourceType, VideoBufferType bufferType)
{
isDisposed = false;
_sourceType = sourceType;
_bufferType = bufferType;
using var request = FFIBridge.Instance.NewRequest<NewVideoSourceRequest>();
var newVideoSource = request.request;
newVideoSource.Resolution = request.TempResource<VideoSourceResolution>();
newVideoSource.Resolution.Width = 1280;
newVideoSource.Resolution.Height = 720;
newVideoSource.Type = VideoSourceType.VideoSourceNative;
using var response = request.Send();
FfiResponse res = response;
_info = res.NewVideoSource.Source.Info;
Handle = FfiHandle.FromOwnedHandle(res.NewVideoSource.Source.Handle);
Handle = null;
}

protected void Init()
{
if (Handle == null)
{
using var request = FFIBridge.Instance.NewRequest<NewVideoSourceRequest>();
var newVideoSource = request.request;
newVideoSource.Resolution = request.TempResource<VideoSourceResolution>();
newVideoSource.Resolution.Width = (uint)GetWidth();
newVideoSource.Resolution.Height = (uint)GetHeight();
newVideoSource.Type = VideoSourceType.VideoSourceNative;
using var response = request.Send();
FfiResponse res = response;
_info = res.NewVideoSource.Source.Info;
Handle = FfiHandle.FromOwnedHandle(res.NewVideoSource.Source.Handle);
}
}

protected TextureFormat GetTextureFormat(VideoBufferType type)
Expand All @@ -59,6 +69,27 @@ protected TextureFormat GetTextureFormat(VideoBufferType type)
return TextureFormat.RGBA32;
case VideoBufferType.Argb:
return TextureFormat.ARGB32;
case VideoBufferType.Bgra:
return TextureFormat.BGRA32;
case VideoBufferType.Rgb24:
return TextureFormat.RGB24;
default:
throw new NotImplementedException("TODO: Add TextureFormat support for type: " + type);
}
}

protected VideoBufferType GetVideoBufferType(TextureFormat type)
{
switch (type)
{
case TextureFormat.RGBA32:
return VideoBufferType.Rgba;
case TextureFormat.ARGB32:
return VideoBufferType.Argb;
case TextureFormat.BGRA32:
return VideoBufferType.Bgra;
case TextureFormat.RGB24:
return VideoBufferType.Rgb24;
default:
throw new NotImplementedException("TODO: Add TextureFormat support for type: " + type);
}
Expand All @@ -70,7 +101,10 @@ protected int GetStrideForBuffer(VideoBufferType type)
{
case VideoBufferType.Rgba:
case VideoBufferType.Argb:
case VideoBufferType.Bgra:
return 4;
case VideoBufferType.Rgb24:
return 3;
default:
throw new NotImplementedException("TODO: Add stride support for type: " + type);
}
Expand Down
30 changes: 19 additions & 11 deletions Runtime/Scripts/ScreenVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,14 @@
using LiveKit.Internal;
using UnityEngine.Rendering;
using Unity.Collections;
using Unity.Collections.LowLevel.Unsafe;
using System.Threading;
using LiveKit.Internal.FFIClients.Requests;
using UnityEngine.Experimental.Rendering;
using UnityEngine.UI;
using System.Threading.Tasks;

namespace LiveKit
{
public class ScreenVideoSource : RtcVideoSource
{
private TextureFormat _textureFormat;

public override int GetWidth()
{
return Screen.width;
Expand All @@ -27,7 +24,7 @@ public override int GetHeight()

public ScreenVideoSource(VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Screen, bufferType)
{
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(bufferType), Allocator.Persistent);
base.Init();
}

public override void Stop()
Expand Down Expand Up @@ -57,13 +54,24 @@ protected override void ReadBuffer()
if (_reading)
return;
_reading = true;
if (_dest == null)
try
{
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
{
var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
var compatibleFormat = SystemInfo.GetCompatibleFormat(targetFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, compatibleFormat);
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
}
ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
}
catch (Exception e)
{
var targetFormat = Utils.GetSupportedGraphicsFormat(SystemInfo.graphicsDeviceType);
_dest = new RenderTexture(GetWidth(), GetHeight(), 0, targetFormat);
Utils.Error(e);
}
ScreenCapture.CaptureScreenshotIntoRenderTexture(_dest as RenderTexture);
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, GetTextureFormat(_bufferType), OnReadback);
}

protected override bool SendFrame()
Expand Down
16 changes: 10 additions & 6 deletions Runtime/Scripts/TextureVideoSource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ namespace LiveKit
{
public class TextureVideoSource : RtcVideoSource
{
TextureFormat _textureFormat;

public Texture Texture { get; }

public override int GetWidth()
Expand All @@ -23,10 +25,9 @@ public override int GetHeight()
public TextureVideoSource(Texture texture, VideoBufferType bufferType = VideoBufferType.Rgba) : base(VideoStreamSource.Texture, bufferType)
{
Texture = texture;
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(bufferType), Allocator.Persistent);
base.Init();
}


~TextureVideoSource()
{
Dispose();
Expand All @@ -38,23 +39,26 @@ protected override void ReadBuffer()
if (_reading)
return;
_reading = true;
var gpuTextureFormat = GetTextureFormat(_bufferType);
if (!SystemInfo.IsFormatSupported(Texture.graphicsFormat, FormatUsage.ReadPixels))
{
if (_dest == null || _dest.width != GetWidth() || _dest.height != GetHeight())
{

var compatibleFormat = SystemInfo.GetCompatibleFormat(Texture.graphicsFormat, FormatUsage.ReadPixels);
_textureFormat = GraphicsFormatUtility.GetTextureFormat(compatibleFormat);
_bufferType = GetVideoBufferType(_textureFormat);
_data = new NativeArray<byte>(GetWidth() * GetHeight() * GetStrideForBuffer(_bufferType), Allocator.Persistent);
_dest = new Texture2D(GetWidth(), GetHeight(), gpuTextureFormat, false);
_dest = new Texture2D(GetWidth(), GetHeight(), _textureFormat, false);
}
Graphics.CopyTexture(Texture, _dest);
}
else
{
_dest = Texture;
_textureFormat = GraphicsFormatUtility.GetTextureFormat(Texture.graphicsFormat);
_bufferType = GetVideoBufferType(_textureFormat);
}

AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, gpuTextureFormat, OnReadback);
AsyncGPUReadback.RequestIntoNativeArray(ref _data, _dest, 0, _textureFormat, OnReadback);
}
}
}
Expand Down

0 comments on commit 175aef8

Please sign in to comment.