From dd43ac0ad6e8e4a9ac5b62fbcbf3558fea26f5d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20CORTIER?= Date: Fri, 1 Sep 2023 10:30:25 -0400 Subject: [PATCH] feat(web): use softbuffer to draw into the canvas Issue: ARC-164 --- Cargo.lock | 179 +++++++++++++++--- Cargo.toml | 7 +- crates/ironrdp-client/Cargo.toml | 4 +- crates/ironrdp-client/src/rdp.rs | 2 +- crates/ironrdp-graphics/src/rdp6/rle.rs | 2 +- crates/ironrdp-session/src/active_stage.rs | 2 +- crates/ironrdp-web/Cargo.toml | 28 +-- crates/ironrdp-web/src/canvas.rs | 86 +++++++++ crates/ironrdp-web/src/lib.rs | 3 +- crates/ironrdp-web/src/session.rs | 160 ++++++---------- web-client/iron-remote-gui/README.md | 1 - web-client/iron-remote-gui/index.html | 2 +- web-client/iron-remote-gui/public/test.html | 2 +- .../src/iron-remote-gui.svelte | 15 -- .../src/services/wasm-bridge.service.ts | 13 +- .../src/lib/login/login.svelte | 18 +- .../lib/remote-screen/remote-screen.svelte | 3 +- .../src/services/server-bridge.service.ts | 2 - .../src/services/services-injector.ts | 12 +- .../src/services/tauri-bridge.service.ts | 61 ------ .../src/services/wasm-bridge.service.ts | 3 - .../iron-svelte-client/svelte.config.js | 9 +- 22 files changed, 337 insertions(+), 277 deletions(-) create mode 100644 crates/ironrdp-web/src/canvas.rs delete mode 100644 web-client/iron-svelte-client/src/services/tauri-bridge.service.ts diff --git a/Cargo.lock b/Cargo.lock index 40a0b9d6a..559bc5e78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -454,6 +454,20 @@ name = "bytemuck" version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", +] [[package]] name = "byteorder" @@ -587,16 +601,16 @@ checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "cocoa" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f425db7937052c684daec3bd6375c8abe2d146dca4b8b143d6db777c39138f3a" +checksum = "f6140449f97a6e97f9511815c5632d84c8aacf8ac271ad77c559218161a1373c" dependencies = [ "bitflags 1.3.2", "block", "cocoa-foundation", "core-foundation", - "core-graphics", - "foreign-types", + "core-graphics 0.23.1", + "foreign-types 0.5.0", "libc", "objc", ] @@ -611,7 +625,7 @@ dependencies = [ "block", "core-foundation", "core-graphics-types", - "foreign-types", + "foreign-types 0.3.2", "libc", "objc", ] @@ -663,7 +677,20 @@ dependencies = [ "bitflags 1.3.2", "core-foundation", "core-graphics-types", - "foreign-types", + "foreign-types 0.3.2", + "libc", +] + +[[package]] +name = "core-graphics" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970a29baf4110c26fedbc7f82107d42c23f7e88e404c4577ed73fe99ff85a212" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-graphics-types", + "foreign-types 0.5.0", "libc", ] @@ -772,6 +799,16 @@ dependencies = [ "subtle 2.4.1", ] +[[package]] +name = "ctor" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b" +dependencies = [ + "quote 1.0.33", + "syn 2.0.29", +] + [[package]] name = "curve25519-dalek" version = "4.0.0" @@ -956,7 +993,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330c60081dcc4c72131f8eb70510f1ac07223e5d4163db481a04a0befcffa412" dependencies = [ - "libloading", + "libloading 0.7.4", ] [[package]] @@ -965,6 +1002,44 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650" +[[package]] +name = "drm" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edf9159ef4bcecd0c5e4cbeb573b8d0037493403d542780dba5d840bbf9df56f" +dependencies = [ + "bitflags 1.3.2", + "bytemuck", + "drm-ffi", + "drm-fourcc", + "nix 0.26.2", +] + +[[package]] +name = "drm-ffi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1352481b7b90e27a8a1bf8ef6b33cf18b98dba7c410e75c24bb3eef2f0d8d525" +dependencies = [ + "drm-sys", + "nix 0.26.2", +] + +[[package]] +name = "drm-fourcc" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aafbcdb8afc29c1a7ee5fbe53b5d62f4565b35a042a662ca9fecd0b54dae6f4" + +[[package]] +name = "drm-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1369f1679d6b706d234c4c1e0613c415c2c74b598a09ad28080ba2474b72e42d" +dependencies = [ + "libc", +] + [[package]] name = "dyn-clone" version = "1.0.13" @@ -1100,15 +1175,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.0.0" @@ -1168,7 +1234,28 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" dependencies = [ - "foreign-types-shared", + "foreign-types-shared 0.1.1", +] + +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared 0.3.1", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.33", + "syn 2.0.29", ] [[package]] @@ -1177,6 +1264,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.0" @@ -2016,6 +2109,7 @@ dependencies = [ "js-sys", "semver", "smallvec", + "softbuffer", "tap", "time 0.3.27", "tracing", @@ -2024,6 +2118,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", + "web-sys", "x509-cert", ] @@ -2104,6 +2199,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d580318f95776505201b28cf98eb1fa5e4be3b689633ba6a3e6cd880ff22d8cb" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "libm" version = "0.2.7" @@ -2248,9 +2353,9 @@ dependencies = [ [[package]] name = "memmap2" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d28bba84adfe6646737845bc5ebbfa2c08424eb1c37e94a1fd2a82adb56a872" +checksum = "f49388d20533534cd19360ad3d6a7dadc885944aa802ba3995040c5ec11288c6" dependencies = [ "libc", ] @@ -2679,7 +2784,7 @@ checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e" dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", - "foreign-types", + "foreign-types 0.3.2", "libc", "once_cell", "openssl-macros", @@ -3866,29 +3971,31 @@ dependencies = [ [[package]] name = "softbuffer" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cc7e34c6782f6fe16e384f25d009ed7b30f70e022d77966769a0366232c4dc4" +source = "git+https://github.com/CBenoit/softbuffer.git?rev=8a9e7f95e054f9af5752682aa5f27890aeb1b094#8a9e7f95e054f9af5752682aa5f27890aeb1b094" dependencies = [ + "as-raw-xcb-connection", "bytemuck", "cfg_aliases", "cocoa", - "core-graphics", - "fastrand 1.9.0", - "foreign-types", + "core-graphics 0.23.1", + "drm", + "drm-sys", + "fastrand", + "foreign-types 0.5.0", "js-sys", "log", - "memmap2 0.6.2", + "memmap2 0.7.1", "nix 0.26.2", "objc", "raw-window-handle", "redox_syscall", + "tiny-xlib", "wasm-bindgen", "wayland-backend", "wayland-client 0.30.2", "wayland-sys 0.30.1", "web-sys", "windows-sys 0.48.0", - "x11-dl", "x11rb", ] @@ -4041,7 +4148,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if 1.0.0", - "fastrand 2.0.0", + "fastrand", "redox_syscall", "rustix", "windows-sys 0.48.0", @@ -4142,6 +4249,18 @@ dependencies = [ "strict-num", ] +[[package]] +name = "tiny-xlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4098d49269baa034a8d1eae9bd63e9fa532148d772121dace3bcd6a6c98eb6d" +dependencies = [ + "as-raw-xcb-connection", + "ctor", + "libloading 0.8.0", + "tracing", +] + [[package]] name = "tinyjson" version = "2.5.1" @@ -5009,7 +5128,7 @@ dependencies = [ "bitflags 1.3.2", "cfg_aliases", "core-foundation", - "core-graphics", + "core-graphics 0.22.3", "dispatch", "instant", "libc", @@ -5091,7 +5210,7 @@ dependencies = [ "as-raw-xcb-connection", "gethostname", "libc", - "libloading", + "libloading 0.7.4", "nix 0.26.2", "once_cell", "winapi", diff --git a/Cargo.toml b/Cargo.toml index 401d9c449..2d95bc9fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,10 +21,10 @@ keywords = ["rdp", "remote-desktop", "network", "client", "protocol"] categories = ["network-programming"] [workspace.dependencies] +ironrdp-acceptor = { version = "0.1", path = "crates/ironrdp-acceptor" } ironrdp-async = { version = "0.1", path = "crates/ironrdp-async" } ironrdp-cliprdr = { version = "0.1", path = "crates/ironrdp-cliprdr" } ironrdp-connector = { version = "0.1", path = "crates/ironrdp-connector" } -ironrdp-acceptor = { version = "0.1", path = "crates/ironrdp-acceptor" } ironrdp-dvc = { version = "0.1", path = "crates/ironrdp-dvc" } ironrdp-error = { version = "0.1", path = "crates/ironrdp-error" } ironrdp-futures = { version = "0.1", path = "crates/ironrdp-futures" } @@ -36,13 +36,13 @@ ironrdp-pdu = { version = "0.1", path = "crates/ironrdp-pdu" } ironrdp-rdcleanpath = { version = "0.1", path = "crates/ironrdp-rdcleanpath" } ironrdp-rdpdr = { version = "0.1", path = "crates/ironrdp-rdpdr" } ironrdp-rdpsnd = { version = "0.1", path = "crates/ironrdp-rdpsnd" } +ironrdp-server = { version = "0.1", path = "crates/ironrdp-server" } ironrdp-session-generators = { path = "crates/ironrdp-session-generators" } ironrdp-session = { version = "0.1", path = "crates/ironrdp-session" } ironrdp-svc = { version = "0.1", path = "crates/ironrdp-svc" } ironrdp-testsuite-core = { path = "crates/ironrdp-testsuite-core" } ironrdp-tls = { version = "0.1", path = "crates/ironrdp-tls" } ironrdp-tokio = { version = "0.1", path = "crates/ironrdp-tokio" } -ironrdp-server = { version = "0.1", path = "crates/ironrdp-server" } ironrdp = { version = "0.5", path = "crates/ironrdp" } expect-test = "1" @@ -63,3 +63,6 @@ lto = true inherits = "release" opt-level = "s" lto = true + +[patch.crates-io] +softbuffer = { git = "https://github.com/CBenoit/softbuffer.git", rev = "8a9e7f95e054f9af5752682aa5f27890aeb1b094" } diff --git a/crates/ironrdp-client/Cargo.toml b/crates/ironrdp-client/Cargo.toml index 3674da80d..90b55d3d1 100644 --- a/crates/ironrdp-client/Cargo.toml +++ b/crates/ironrdp-client/Cargo.toml @@ -33,9 +33,9 @@ ironrdp-tls.workspace = true ironrdp-tokio.workspace = true sspi = { workspace = true, features = ["network_client", "dns_resolver"] } # enable additional features -# GUI -softbuffer = "0.3" +# Windowing and rendering winit = "0.28" +softbuffer = "0.3" # CLI clap = { version = "4.2", features = ["derive", "cargo"] } diff --git a/crates/ironrdp-client/src/rdp.rs b/crates/ironrdp-client/src/rdp.rs index 2cf439a7f..d4cafb923 100644 --- a/crates/ironrdp-client/src/rdp.rs +++ b/crates/ironrdp-client/src/rdp.rs @@ -208,7 +208,6 @@ async fn active_session( }) .map_err(|e| session::custom_err!("event_loop_proxy", e))?; } - ActiveStageOutput::Terminate => break 'outer, ActiveStageOutput::PointerDefault => { event_loop_proxy .send_event(RdpOutputEvent::PointerDefault) @@ -224,6 +223,7 @@ async fn active_session( .send_event(RdpOutputEvent::PointerPosition { x, y }) .map_err(|e| session::custom_err!("event_loop_proxy", e))?; } + ActiveStageOutput::Terminate => break 'outer, } } } diff --git a/crates/ironrdp-graphics/src/rdp6/rle.rs b/crates/ironrdp-graphics/src/rdp6/rle.rs index cf6bed7e5..352f21f84 100644 --- a/crates/ironrdp-graphics/src/rdp6/rle.rs +++ b/crates/ironrdp-graphics/src/rdp6/rle.rs @@ -379,7 +379,7 @@ mod tests { decompress_8bpp_plane(src, dst.as_mut_slice(), width, height) } - pub fn compress(src: &[u8], dst: &mut Vec, width: usize, height: usize) -> Result { + pub fn compress(src: &[u8], dst: &mut [u8], width: usize, height: usize) -> Result { compress_8bpp_plane(src.iter().copied(), &mut WriteCursor::new(dst), width, height) } diff --git a/crates/ironrdp-session/src/active_stage.rs b/crates/ironrdp-session/src/active_stage.rs index 199a8094e..2bbd0d283 100644 --- a/crates/ironrdp-session/src/active_stage.rs +++ b/crates/ironrdp-session/src/active_stage.rs @@ -154,8 +154,8 @@ impl ActiveStage { pub enum ActiveStageOutput { ResponseFrame(Vec), GraphicsUpdate(InclusiveRectangle), - Terminate, PointerDefault, PointerHidden, PointerPosition { x: usize, y: usize }, + Terminate, } diff --git a/crates/ironrdp-web/Cargo.toml b/crates/ironrdp-web/Cargo.toml index 06975b4bf..cd03e1527 100644 --- a/crates/ironrdp-web/Cargo.toml +++ b/crates/ironrdp-web/Cargo.toml @@ -29,12 +29,16 @@ ironrdp-futures.workspace = true ironrdp-rdcleanpath.workspace = true # WASM -wasm-bindgen = "0.2.84" -wasm-bindgen-futures = "0.4.34" -js-sys = "0.3.61" -gloo-net = { version = "0.2.6", default-features = false, features = ["websocket", "http"] } -gloo-timers = { version = "0.2.6", default-features = false, features = ["futures"] } -tracing-web = "0.1.2" +wasm-bindgen = "0.2" +wasm-bindgen-futures = "0.4" +web-sys = { version = "0.3", features = ["HtmlCanvasElement"] } +js-sys = "0.3" +gloo-net = { version = "0.2", default-features = false, features = ["websocket", "http"] } +gloo-timers = { version = "0.2", default-features = false, features = ["futures"] } +tracing-web = "0.1" + +# Rendering +softbuffer = { version = "0.3", default-features = false } # Enable WebAssembly support for a few crates getrandom = { version = "0.2", features = ["js"] } @@ -45,7 +49,7 @@ time = { version = "0.3", features = ["wasm-bindgen"] } # logging them with `console.error`. This is great for development, but requires # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for # code size when deploying. -console_error_panic_hook = { version = "0.1.7", optional = true } +console_error_panic_hook = { version = "0.1", optional = true } # Async futures-util = { version = "0.3", features = ["sink", "io"] } @@ -53,12 +57,12 @@ futures-channel = "0.3" # Logging tracing.workspace = true -tracing-subscriber = { version = "0.3.16", features = ["time"] } +tracing-subscriber = { version = "0.3", features = ["time"] } # Utils anyhow = "1" -smallvec = "1.10.0" -x509-cert = { version = "0.2.1", default-features = false, features = ["std"] } -tap = "1.0.1" +smallvec = "1.10" +x509-cert = { version = "0.2", default-features = false, features = ["std"] } +tap = "1" semver = "1" -url = "2.4.0" +url = "2" diff --git a/crates/ironrdp-web/src/canvas.rs b/crates/ironrdp-web/src/canvas.rs new file mode 100644 index 000000000..6fccdd731 --- /dev/null +++ b/crates/ironrdp-web/src/canvas.rs @@ -0,0 +1,86 @@ +use std::num::NonZeroU32; + +use ironrdp::pdu::geometry::{InclusiveRectangle, Rectangle as _}; +use web_sys::HtmlCanvasElement; + +pub struct Canvas { + width: u32, + surface: softbuffer::Surface, +} + +impl Canvas { + pub fn new(render_canvas: HtmlCanvasElement, width: u32, height: u32) -> anyhow::Result { + render_canvas.set_width(width); + render_canvas.set_height(height); + + #[cfg(target_arch = "wasm32")] + let mut surface = { + use softbuffer::SurfaceExtWeb as _; + softbuffer::Surface::from_canvas(render_canvas).expect("surface") + }; + + #[cfg(not(target_arch = "wasm32"))] + let mut surface = { + fn stub(_: HtmlCanvasElement) -> softbuffer::Surface { + unimplemented!() + } + + stub(render_canvas) + }; + + surface + .resize(NonZeroU32::new(width).unwrap(), NonZeroU32::new(height).unwrap()) + .expect("surface resize"); + + Ok(Self { width, surface }) + } + + pub fn draw(&mut self, buffer: &[u8], region: InclusiveRectangle) -> anyhow::Result<()> { + let region_width = region.width(); + let region_height = region.height(); + + let mut src = buffer.chunks_exact(4).map(|pixel| { + let r = pixel[0]; + let g = pixel[1]; + let b = pixel[2]; + u32::from_be_bytes([0, r, g, b]) + }); + + let mut dst = self.surface.buffer_mut().expect("surface buffer"); + + { + // Copy src into dst + + let region_top_usize = usize::from(region.top); + let region_height_usize = usize::from(region_height); + let region_left_usize = usize::from(region.left); + let region_width_usize = usize::from(region_width); + + for dst_row in dst + .chunks_exact_mut(self.width as usize) + .skip(region_top_usize) + .take(region_height_usize) + { + let src_row = src.by_ref().take(region_width_usize); + + dst_row + .iter_mut() + .skip(region_left_usize) + .take(region_width_usize) + .zip(src_row) + .for_each(|(dst, src)| *dst = src); + } + } + + let damage_rect = softbuffer::Rect { + x: u32::from(region.left), + y: u32::from(region.top), + width: NonZeroU32::new(u32::from(region_width)).unwrap(), + height: NonZeroU32::new(u32::from(region_height)).unwrap(), + }; + + dst.present_with_damage(&[damage_rect]).expect("buffer present"); + + Ok(()) + } +} diff --git a/crates/ironrdp-web/src/lib.rs b/crates/ironrdp-web/src/lib.rs index 2d5d6b617..cea3cc85a 100644 --- a/crates/ironrdp-web/src/lib.rs +++ b/crates/ironrdp-web/src/lib.rs @@ -3,6 +3,7 @@ #[macro_use] extern crate tracing; +mod canvas; mod error; mod image; mod input; @@ -27,7 +28,7 @@ pub fn ironrdp_init(log_level: &str) { // // For more details see // https://github.com/rustwasm/console_error_panic_hook#readme - #[cfg(feature = "console_error_panic_hook")] + #[cfg(feature = "panic_hook")] console_error_panic_hook::set_once(); if let Ok(level) = log_level.parse::() { diff --git a/crates/ironrdp-web/src/session.rs b/crates/ironrdp-web/src/session.rs index 94fa49080..98ee3bb27 100644 --- a/crates/ironrdp-web/src/session.rs +++ b/crates/ironrdp-web/src/session.rs @@ -5,12 +5,11 @@ use std::time::Duration; use anyhow::Context as _; use futures_channel::mpsc; use futures_util::io::{ReadHalf, WriteHalf}; -use futures_util::{select, AsyncReadExt as _, AsyncWriteExt as _, FutureExt, StreamExt as _}; +use futures_util::{select, AsyncReadExt as _, AsyncWriteExt as _, FutureExt as _, StreamExt as _}; use gloo_net::websocket; use gloo_net::websocket::futures::WebSocket; use ironrdp::connector::{self, ClientConnector}; use ironrdp::graphics::image_processing::PixelFormat; -use ironrdp::pdu::geometry::{InclusiveRectangle, Rectangle as _}; use ironrdp::pdu::input::fast_path::FastPathInputEvent; use ironrdp::pdu::write_buf::WriteBuf; use ironrdp::session::image::DecodedImage; @@ -18,9 +17,11 @@ use ironrdp::session::{ActiveStage, ActiveStageOutput}; use tap::prelude::*; use wasm_bindgen::prelude::*; use wasm_bindgen_futures::spawn_local; +use web_sys::HtmlCanvasElement; +use crate::canvas::Canvas; use crate::error::{IronRdpError, IronRdpErrorKind}; -use crate::image::{extract_partial_image, RectInfo}; +use crate::image::extract_partial_image; use crate::input::InputTransaction; use crate::network_client::WasmNetworkClientFactory; use crate::websocket::WebSocketCompat; @@ -44,8 +45,7 @@ struct SessionBuilderInner { client_name: String, desktop_size: DesktopSize, - update_callback: Option, - update_callback_context: Option, + render_canvas: Option, hide_pointer_callback: Option, hide_pointer_callback_context: Option, show_pointer_callback: Option, @@ -67,9 +67,8 @@ impl Default for SessionBuilderInner { width: DEFAULT_WIDTH, height: DEFAULT_HEIGHT, }, - update_callback: None, - update_callback_context: None, + render_canvas: None, hide_pointer_callback: None, hide_pointer_callback_context: None, show_pointer_callback: None, @@ -136,15 +135,9 @@ impl SessionBuilder { self.clone() } - /// Required - pub fn update_callback(&self, callback: js_sys::Function) -> SessionBuilder { - self.0.borrow_mut().update_callback = Some(callback); - self.clone() - } - - /// Required - pub fn update_callback_context(&self, context: JsValue) -> SessionBuilder { - self.0.borrow_mut().update_callback_context = Some(context); + /// Optional + pub fn render_canvas(&self, canvas: HtmlCanvasElement) -> SessionBuilder { + self.0.borrow_mut().render_canvas = Some(canvas); self.clone() } @@ -183,8 +176,7 @@ impl SessionBuilder { pcb, client_name, desktop_size, - update_callback, - update_callback_context, + render_canvas, hide_pointer_callback, hide_pointer_callback_context, show_pointer_callback, @@ -193,29 +185,34 @@ impl SessionBuilder { { let inner = self.0.borrow(); - username = inner.username.clone().expect("username"); - destination = inner.destination.clone().expect("destination"); + username = inner.username.clone().context("username missing")?; + destination = inner.destination.clone().context("destination missing")?; server_domain = inner.server_domain.clone(); - password = inner.password.clone().expect("password"); - proxy_address = inner.proxy_address.clone().expect("proxy_address"); - auth_token = inner.auth_token.clone().expect("auth_token"); + password = inner.password.clone().context("password missing")?; + proxy_address = inner.proxy_address.clone().context("proxy_address missing")?; + auth_token = inner.auth_token.clone().context("auth_token missing")?; pcb = inner.pcb.clone(); client_name = inner.client_name.clone(); desktop_size = inner.desktop_size.clone(); - update_callback = inner.update_callback.clone().expect("update_callback"); - update_callback_context = inner.update_callback_context.clone().expect("update_callback_context"); + render_canvas = inner.render_canvas.clone().context("render_canvas missing")?; - hide_pointer_callback = inner.hide_pointer_callback.clone().expect("hide_pointer_callback"); + hide_pointer_callback = inner + .hide_pointer_callback + .clone() + .context("hide_pointer_callback missing")?; hide_pointer_callback_context = inner .hide_pointer_callback_context .clone() - .expect("show_pointer_callback_context"); - show_pointer_callback = inner.show_pointer_callback.clone().expect("hide_pointer_callback"); + .context("show_pointer_callback_context missing")?; + show_pointer_callback = inner + .show_pointer_callback + .clone() + .context("hide_pointer_callback missing")?; show_pointer_callback_context = inner .show_pointer_callback_context .clone() - .expect("show_pointer_callback_context"); + .context("show_pointer_callback_context missing")?; } info!("Connect to RDP host"); @@ -268,8 +265,7 @@ impl SessionBuilder { writer_tx, input_events_tx, - update_callback, - update_callback_context, + render_canvas, hide_pointer_callback, hide_pointer_callback_context, show_pointer_callback, @@ -291,8 +287,7 @@ pub struct Session { writer_tx: mpsc::UnboundedSender>, input_events_tx: mpsc::UnboundedSender, - update_callback: js_sys::Function, - update_callback_context: JsValue, + render_canvas: HtmlCanvasElement, hide_pointer_callback: js_sys::Function, hide_pointer_callback_context: JsValue, show_pointer_callback: js_sys::Function, @@ -327,32 +322,55 @@ impl Session { let mut framed = ironrdp_futures::SingleThreadedFuturesFramed::new(rdp_reader); + debug!("Initialize canvas"); + + let mut gui = Canvas::new( + self.render_canvas.clone(), + u32::from(connection_result.desktop_size.width), + u32::from(connection_result.desktop_size.height), + ) + .context("canvas initialization")?; + + debug!("Canvas initialized"); + info!("Start RDP session"); - let mut image = DecodedImage::new(PixelFormat::RgbA32, self.desktop_size.width, self.desktop_size.height); + let mut image = DecodedImage::new( + PixelFormat::RgbA32, + connection_result.desktop_size.width, + connection_result.desktop_size.height, + ); let mut active_stage = ActiveStage::new(connection_result, None); - let mut frame_id = 0; 'outer: loop { let outputs = select! { - incoming_pdu = framed.read_pdu().fuse() => { - let (action, frame) = incoming_pdu.context("read next frame")?; + frame = framed.read_pdu().fuse() => { + let (action, payload) = frame.context("read frame")?; + trace!(?action, frame_length = payload.len(), "Frame received"); - active_stage - .process(&mut image, action, &frame) - .context("Active stage processing")? + active_stage.process(&mut image, action, &payload)? } input_events = fastpath_input_events.next() => { let events = input_events.context("read next fastpath input events")?; - active_stage.process_fastpath_input(&mut image, &events) - .context("Fast path input events processing")? + active_stage.process_fastpath_input(&mut image, &events).context("Fast path input events processing")? } }; for out in outputs { match out { + ActiveStageOutput::ResponseFrame(frame) => { + // PERF: unnecessary copy + self.writer_tx + .unbounded_send(frame.to_vec()) + .context("Send frame to writer task")?; + } + ActiveStageOutput::GraphicsUpdate(region) => { + // PERF: some copies and conversion could be optimized + let (region, buffer) = extract_partial_image(&image, region); + gui.draw(&buffer, region).context("draw updated region")?; + } ActiveStageOutput::PointerDefault => { let _ret = self .show_pointer_callback @@ -368,26 +386,6 @@ impl Session { ActiveStageOutput::PointerPosition { .. } => { // Not applicable for web } - ActiveStageOutput::ResponseFrame(frame) => { - // PERF: unnecessary copy - self.writer_tx - .unbounded_send(frame.to_vec()) - .context("Send frame to writer task")?; - } - ActiveStageOutput::GraphicsUpdate(updated_region) => { - let (partial_image_rectangle, partial_image) = extract_partial_image(&image, updated_region); - - send_update_rectangle( - &self.update_callback, - &self.update_callback_context, - frame_id, - partial_image_rectangle, - partial_image, - ) - .context("Failed to send update rectangle")?; - - frame_id += 1; - } ActiveStageOutput::Terminate => break 'outer, } } @@ -499,44 +497,6 @@ fn build_config( } } -fn send_update_rectangle( - update_callback: &js_sys::Function, - callback_context: &JsValue, - frame_id: usize, - region: InclusiveRectangle, - buffer: Vec, -) -> anyhow::Result<()> { - use js_sys::Uint8ClampedArray; - - let top = region.top; - let left = region.left; - let right = region.right; - let bottom = region.bottom; - let width = region.width(); - let height = region.height(); - - let update_rect = RectInfo { - frame_id, - top, - left, - right, - bottom, - width, - height, - }; - let update_rect = JsValue::from(update_rect); - - let js_array = Uint8ClampedArray::new_with_length(buffer.len() as u32); - js_array.copy_from(&buffer); - let js_array = JsValue::from(js_array); - - let _ret = update_callback - .call2(callback_context, &update_rect, &js_array) - .map_err(|e| anyhow::Error::msg(format!("update callback failed: {e:?}")))?; - - Ok(()) -} - async fn writer_task(rx: mpsc::UnboundedReceiver>, rdp_writer: WriteHalf) { debug!("writer task started"); diff --git a/web-client/iron-remote-gui/README.md b/web-client/iron-remote-gui/README.md index 92594bc5e..01c1af3fa 100644 --- a/web-client/iron-remote-gui/README.md +++ b/web-client/iron-remote-gui/README.md @@ -46,7 +46,6 @@ You can add some parameters for default initialization on the component ` Note that due to a limitation of the framework all parameters need to be lower-cased. - `scale`: The scaling behavior of the distant screen. Can be `fit`, `real` or `full`. Default is `real`; -- `targetplatform`: Can be `web` or `native`. Default is `web`. - `verbose`: Show logs from `iron-remote-gui`. `true` or `false`. Default is `false`. - `debugwasm`: Show debug info from web assembly. Can be `"OFF"`, `"ERROR"`, `"WARN"`, `"INFO"`, `"DEBUG"`, `"TRACE"`. Default is `"OFF"`. - `flexcentre`: Helper to force `iron-remote-gui` a flex and centering the content automatically. Otherwise, you need to manage manually. Default is `true`. diff --git a/web-client/iron-remote-gui/index.html b/web-client/iron-remote-gui/index.html index e2223d5f0..6d50f0dc2 100644 --- a/web-client/iron-remote-gui/index.html +++ b/web-client/iron-remote-gui/index.html @@ -7,7 +7,7 @@ - + - +