From dc4ae445254f75432db0425dca9f11f59dbbad33 Mon Sep 17 00:00:00 2001 From: Mads Marquart Date: Wed, 4 Sep 2024 15:48:11 +0200 Subject: [PATCH] Add support for iOS/tvOS/watchOS/visionOS (#234) Add support for iOS / UIKit, and clean up CoreGraphics impl --- .github/workflows/ci.yml | 5 +- CHANGELOG.md | 3 + Cargo.toml | 16 +- README.md | 2 +- src/backend_dispatch.rs | 4 +- src/backends/cg.rs | 306 ++++++++++++++++++++++++++++++--------- src/backends/mod.rs | 2 +- src/lib.rs | 5 +- 8 files changed, 264 insertions(+), 79 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89fd4074..125a112e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - { target: x86_64-unknown-redox, os: ubuntu-latest, } - { target: x86_64-unknown-freebsd, os: ubuntu-latest, } - { target: x86_64-unknown-netbsd, os: ubuntu-latest, options: --no-default-features, features: "x11,x11-dlopen,wayland,wayland-dlopen" } - - { target: x86_64-apple-darwin, os: macos-latest, } + - { target: aarch64-apple-darwin, os: macos-latest, } - { target: wasm32-unknown-unknown, os: ubuntu-latest, } exclude: # Orbital doesn't follow MSRV @@ -56,6 +56,9 @@ jobs: include: - rust_version: nightly platform: { target: wasm32-unknown-unknown, os: ubuntu-latest, options: "-Zbuild-std=panic_abort,std", rustflags: "-Ctarget-feature=+atomics,+bulk-memory" } + # Mac Catalyst is only Tier 2 since Rust 1.81 + - rust_version: 'nightly' + platform: { target: aarch64-apple-ios-macabi, os: macos-latest } env: RUST_BACKTRACE: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 3576f15d..c07d2ff1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Unreleased +- Added support for iOS, tvOS, watchOS and visionOS (UIKit). +- Redo the way surfaces work on macOS to work directly with layers, which will allow initializing directly from a `CALayer` in the future. + # 0.4.5 - Make the `wayland-sys` dependency optional. (#223) diff --git a/Cargo.toml b/Cargo.toml index 8cbc88c3..abc1fb71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,14 +45,20 @@ x11rb = { version = "0.13.0", features = ["allow-unsafe-code", "shm"], optional version = "0.59.0" features = ["Win32_Graphics_Gdi", "Win32_UI_Shell", "Win32_UI_WindowsAndMessaging", "Win32_Foundation"] -[target.'cfg(target_os = "macos")'.dependencies] +[target.'cfg(target_vendor = "apple")'.dependencies] bytemuck = { version = "1.12.3", features = ["extern_crate_alloc"] } core-graphics = "0.24.0" foreign-types = "0.5.0" -objc2 = "0.5.1" -objc2-foundation = { version = "0.2.0", features = ["dispatch", "NSThread"] } -objc2-app-kit = { version = "0.2.0", features = ["NSResponder", "NSView", "NSWindow"] } -objc2-quartz-core = { version = "0.2.0", features = ["CALayer", "CATransaction"] } +objc2 = "0.5.2" +objc2-foundation = { version = "0.2.2", features = [ + "NSDictionary", + "NSGeometry", + "NSKeyValueObserving", + "NSString", + "NSThread", + "NSValue", +] } +objc2-quartz-core = { version = "0.2.2", features = ["CALayer", "CATransaction"] } [target.'cfg(target_arch = "wasm32")'.dependencies] js-sys = "0.3.63" diff --git a/README.md b/README.md index eccfe26c..599df21c 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ For now, the priority for new platforms is: |Android NDK|❌| | AppKit |✅| | Orbital |✅| -| UIKit |❌| +| UIKit |✅| | Wayland |✅| | Web |✅| | Win32 |✅| diff --git a/src/backend_dispatch.rs b/src/backend_dispatch.rs index d8d23aa2..208f82cd 100644 --- a/src/backend_dispatch.rs +++ b/src/backend_dispatch.rs @@ -186,8 +186,8 @@ make_dispatch! { Kms(Arc>, backends::kms::KmsImpl, backends::kms::BufferImpl<'a, D, W>), #[cfg(target_os = "windows")] Win32(D, backends::win32::Win32Impl, backends::win32::BufferImpl<'a, D, W>), - #[cfg(target_os = "macos")] - CG(D, backends::cg::CGImpl, backends::cg::BufferImpl<'a, D, W>), + #[cfg(target_vendor = "apple")] + CoreGraphics(D, backends::cg::CGImpl, backends::cg::BufferImpl<'a, D, W>), #[cfg(target_arch = "wasm32")] Web(backends::web::WebDisplayImpl, backends::web::WebImpl, backends::web::BufferImpl<'a, D, W>), #[cfg(target_os = "redox")] diff --git a/src/backends/cg.rs b/src/backends/cg.rs index 4a5db6a9..ba957d85 100644 --- a/src/backends/cg.rs +++ b/src/backends/cg.rs @@ -7,18 +7,23 @@ use core_graphics::base::{ use core_graphics::color_space::CGColorSpace; use core_graphics::data_provider::CGDataProvider; use core_graphics::image::CGImage; -use objc2::runtime::AnyObject; -use raw_window_handle::{HasDisplayHandle, HasWindowHandle, RawWindowHandle}; - use foreign_types::ForeignType; -use objc2::msg_send; -use objc2::rc::Id; -use objc2_app_kit::{NSAutoresizingMaskOptions, NSView, NSWindow}; -use objc2_foundation::{MainThreadBound, MainThreadMarker}; +use objc2::rc::Retained; +use objc2::runtime::{AnyObject, Bool}; +use objc2::{declare_class, msg_send, msg_send_id, mutability, ClassType, DeclaredClass}; +use objc2_foundation::{ + ns_string, CGPoint, MainThreadMarker, NSDictionary, NSKeyValueChangeKey, + NSKeyValueChangeNewKey, NSKeyValueObservingOptions, NSNumber, NSObject, + NSObjectNSKeyValueObserverRegistration, NSString, NSValue, +}; use objc2_quartz_core::{kCAGravityTopLeft, CALayer, CATransaction}; +use raw_window_handle::{HasDisplayHandle, HasWindowHandle, RawWindowHandle}; +use std::ffi::c_void; use std::marker::PhantomData; use std::num::NonZeroU32; +use std::ops::Deref; +use std::ptr; use std::sync::Arc; struct Buffer(Vec); @@ -29,18 +34,112 @@ impl AsRef<[u8]> for Buffer { } } +declare_class!( + struct Observer; + + unsafe impl ClassType for Observer { + type Super = NSObject; + type Mutability = mutability::InteriorMutable; + const NAME: &'static str = "SoftbufferObserver"; + } + + impl DeclaredClass for Observer { + type Ivars = Retained; + } + + // NSKeyValueObserving + unsafe impl Observer { + #[method(observeValueForKeyPath:ofObject:change:context:)] + fn observe_value( + &self, + key_path: Option<&NSString>, + _object: Option<&AnyObject>, + change: Option<&NSDictionary>, + _context: *mut c_void, + ) { + self.update(key_path, change); + } + } +); + +// SAFETY: The `CALayer` that the observer contains is thread safe. +unsafe impl Send for Observer {} +unsafe impl Sync for Observer {} + +impl Observer { + fn new(layer: &CALayer) -> Retained { + let this = Self::alloc().set_ivars(layer.retain()); + unsafe { msg_send_id![super(this), init] } + } + + fn update( + &self, + key_path: Option<&NSString>, + change: Option<&NSDictionary>, + ) { + let layer = self.ivars(); + + let change = + change.expect("requested a change dictionary in `addObserver`, but none was provided"); + let new = change + .get(unsafe { NSKeyValueChangeNewKey }) + .expect("requested change dictionary did not contain `NSKeyValueChangeNewKey`"); + + // NOTE: Setting these values usually causes a quarter second animation to occur, which is + // undesirable. + // + // However, since we're setting them inside an observer, there already is a transaction + // ongoing, and as such we don't need to wrap this in a `CATransaction` ourselves. + + if key_path == Some(ns_string!("contentsScale")) { + let new = unsafe { &*(new as *const AnyObject as *const NSNumber) }; + let scale_factor = new.as_cgfloat(); + + // Set the scale factor of the layer to match the root layer when it changes (e.g. if + // moved to a different monitor, or monitor settings changed). + layer.setContentsScale(scale_factor); + } else if key_path == Some(ns_string!("bounds")) { + let new = unsafe { &*(new as *const AnyObject as *const NSValue) }; + let bounds = new.get_rect().expect("new bounds value was not CGRect"); + + // Set `bounds` and `position` so that the new layer is inside the superlayer. + // + // This differs from just setting the `bounds`, as it also takes into account any + // translation that the superlayer may have that we'd want to preserve. + layer.setFrame(bounds); + } else { + panic!("unknown observed keypath {key_path:?}"); + } + } +} + pub struct CGImpl { - layer: MainThreadBound>, - window: MainThreadBound>, + /// Our layer. + layer: SendCALayer, + /// The layer that our layer was created from. + /// + /// Can also be retrieved from `layer.superlayer()`. + root_layer: SendCALayer, + observer: Retained, color_space: SendCGColorSpace, - size: Option<(NonZeroU32, NonZeroU32)>, + /// The width of the underlying buffer. + width: usize, + /// The height of the underlying buffer. + height: usize, window_handle: W, _display: PhantomData, } -// TODO(madsmtm): Expose this in `objc2_app_kit`. -fn set_layer(view: &NSView, layer: &CALayer) { - unsafe { msg_send![view, setLayer: layer] } +impl Drop for CGImpl { + fn drop(&mut self) { + // SAFETY: Registered in `new`, must be removed before the observer is deallocated. + unsafe { + self.root_layer + .removeObserver_forKeyPath(&self.observer, ns_string!("contentsScale")); + self.root_layer + .removeObserver_forKeyPath(&self.observer, ns_string!("bounds")); + } + } } impl SurfaceInterface for CGImpl { @@ -48,46 +147,118 @@ impl SurfaceInterface for CGImpl< type Buffer<'a> = BufferImpl<'a, D, W> where Self: 'a; fn new(window_src: W, _display: &D) -> Result> { - let raw = window_src.window_handle()?.as_raw(); - let handle = match raw { - RawWindowHandle::AppKit(handle) => handle, + // `NSView`/`UIView` can only be accessed from the main thread. + let _mtm = MainThreadMarker::new().ok_or(SoftBufferError::PlatformError( + Some("can only access Core Graphics handles from the main thread".to_string()), + None, + ))?; + + let root_layer = match window_src.window_handle()?.as_raw() { + RawWindowHandle::AppKit(handle) => { + // SAFETY: The pointer came from `WindowHandle`, which ensures that the + // `AppKitWindowHandle` contains a valid pointer to an `NSView`. + // + // We use `NSObject` here to avoid importing `objc2-app-kit`. + let view: &NSObject = unsafe { handle.ns_view.cast().as_ref() }; + + // Force the view to become layer backed + let _: () = unsafe { msg_send![view, setWantsLayer: Bool::YES] }; + + // SAFETY: `-[NSView layer]` returns an optional `CALayer` + let layer: Option> = unsafe { msg_send_id![view, layer] }; + layer.expect("failed making the view layer-backed") + } + RawWindowHandle::UiKit(handle) => { + // SAFETY: The pointer came from `WindowHandle`, which ensures that the + // `UiKitWindowHandle` contains a valid pointer to an `UIView`. + // + // We use `NSObject` here to avoid importing `objc2-ui-kit`. + let view: &NSObject = unsafe { handle.ui_view.cast().as_ref() }; + + // SAFETY: `-[UIView layer]` returns `CALayer` + let layer: Retained = unsafe { msg_send_id![view, layer] }; + layer + } _ => return Err(InitError::Unsupported(window_src)), }; - // `NSView` can only be accessed from the main thread. - let mtm = MainThreadMarker::new().ok_or(SoftBufferError::PlatformError( - Some("can only access AppKit / macOS handles from the main thread".to_string()), - None, - ))?; - let view = handle.ns_view.as_ptr(); - // SAFETY: The pointer came from `WindowHandle`, which ensures that - // the `AppKitWindowHandle` contains a valid pointer to an `NSView`. - // Unwrap is fine, since the pointer came from `NonNull`. - let view: Id = unsafe { Id::retain(view.cast()) }.unwrap(); + // Add a sublayer, to avoid interfering with the root layer, since setting the contents of + // e.g. a view-controlled layer is brittle. let layer = CALayer::new(); - let subview = unsafe { NSView::initWithFrame(mtm.alloc(), view.frame()) }; - layer.setContentsGravity(unsafe { kCAGravityTopLeft }); - layer.setNeedsDisplayOnBoundsChange(false); - set_layer(&subview, &layer); + root_layer.addSublayer(&layer); + + // Set the anchor point and geometry. Softbuffer's uses a coordinate system with the origin + // in the top-left corner. + // + // NOTE: This doesn't really matter unless we start modifying the `position` of our layer + // ourselves, but it's nice to have in place. + layer.setAnchorPoint(CGPoint::new(0.0, 0.0)); + layer.setGeometryFlipped(true); + + // Do not use auto-resizing mask. + // + // This is done to work around a bug in macOS 14 and above, where views using auto layout + // may end up setting fractional values as the bounds, and that in turn doesn't propagate + // properly through the auto-resizing mask and with contents gravity. + // + // Instead, we keep the bounds of the layer in sync with the root layer using an observer, + // see below. + // + // layer.setAutoresizingMask(kCALayerHeightSizable | kCALayerWidthSizable); + + let observer = Observer::new(&layer); + // Observe changes to the root layer's bounds and scale factor, and apply them to our layer. + // + // The previous implementation updated the scale factor inside `resize`, but this works + // poorly with transactions, and is generally inefficient. Instead, we update the scale + // factor only when needed because the super layer's scale factor changed. + // + // Note that inherent in this is an explicit design decision: We control the `bounds` and + // `contentsScale` of the layer directly, and instead let the `resize` call that the user + // controls only be the size of the underlying buffer. + // + // SAFETY: Observer deregistered in `Drop` before the observer object is deallocated. unsafe { - subview.setAutoresizingMask(NSAutoresizingMaskOptions( - NSAutoresizingMaskOptions::NSViewWidthSizable.0 - | NSAutoresizingMaskOptions::NSViewHeightSizable.0, - )) - }; + root_layer.addObserver_forKeyPath_options_context( + &observer, + ns_string!("contentsScale"), + NSKeyValueObservingOptions::NSKeyValueObservingOptionNew + | NSKeyValueObservingOptions::NSKeyValueObservingOptionInitial, + ptr::null_mut(), + ); + root_layer.addObserver_forKeyPath_options_context( + &observer, + ns_string!("bounds"), + NSKeyValueObservingOptions::NSKeyValueObservingOptionNew + | NSKeyValueObservingOptions::NSKeyValueObservingOptionInitial, + ptr::null_mut(), + ); + } - let window = view.window().ok_or(SoftBufferError::PlatformError( - Some("view must be inside a window".to_string()), - None, - ))?; + // Set the content so that it is placed in the top-left corner if it does not have the same + // size as the surface itself. + // + // TODO(madsmtm): Consider changing this to `kCAGravityResize` to stretch the content if + // resized to something that doesn't fit, see #177. + layer.setContentsGravity(unsafe { kCAGravityTopLeft }); - unsafe { view.addSubview(&subview) }; + // Initialize color space here, to reduce work later on. let color_space = CGColorSpace::create_device_rgb(); + + // Grab initial width and height from the layer (whose properties have just been initialized + // by the observer using `NSKeyValueObservingOptionInitial`). + let size = layer.bounds().size; + let scale_factor = layer.contentsScale(); + let width = (size.width * scale_factor) as usize; + let height = (size.height * scale_factor) as usize; + Ok(Self { - layer: MainThreadBound::new(layer, mtm), - window: MainThreadBound::new(window, mtm), + layer: SendCALayer(layer), + root_layer: SendCALayer(root_layer), + observer, color_space: SendCGColorSpace(color_space), - size: None, + width, + height, _display: PhantomData, window_handle: window_src, }) @@ -99,17 +270,14 @@ impl SurfaceInterface for CGImpl< } fn resize(&mut self, width: NonZeroU32, height: NonZeroU32) -> Result<(), SoftBufferError> { - self.size = Some((width, height)); + self.width = width.get() as usize; + self.height = height.get() as usize; Ok(()) } fn buffer_mut(&mut self) -> Result, SoftBufferError> { - let (width, height) = self - .size - .expect("Must set size of surface before calling `buffer_mut()`"); - Ok(BufferImpl { - buffer: vec![0; width.get() as usize * height.get() as usize], + buffer: vec![0; self.width * self.height], imp: self, }) } @@ -137,41 +305,31 @@ impl<'a, D: HasDisplayHandle, W: HasWindowHandle> BufferInterface for BufferImpl fn present(self) -> Result<(), SoftBufferError> { let data_provider = CGDataProvider::from_buffer(Arc::new(Buffer(self.buffer))); - let (width, height) = self.imp.size.unwrap(); + let image = CGImage::new( - width.get() as usize, - height.get() as usize, + self.imp.width, + self.imp.height, 8, 32, - (width.get() * 4) as usize, + self.imp.width * 4, &self.imp.color_space.0, kCGBitmapByteOrder32Little | kCGImageAlphaNoneSkipFirst, &data_provider, false, kCGRenderingIntentDefault, ); - - // TODO: Use run_on_main() instead. - let mtm = MainThreadMarker::new().ok_or(SoftBufferError::PlatformError( - Some("can only access AppKit / macOS handles from the main thread".to_string()), - None, - ))?; + let contents = unsafe { (image.as_ptr() as *mut AnyObject).as_ref() }; // The CALayer has a default action associated with a change in the layer contents, causing // a quarter second fade transition to happen every time a new buffer is applied. This can - // be mitigated by wrapping the operation in a transaction and disabling all actions. + // be avoided by wrapping the operation in a transaction and disabling all actions. CATransaction::begin(); CATransaction::setDisableActions(true); - let layer = self.imp.layer.get(mtm); - layer.setContentsScale(self.imp.window.get(mtm).backingScaleFactor()); - - unsafe { - layer.setContents((image.as_ptr() as *mut AnyObject).as_ref()); - }; + // SAFETY: The contents is `CGImage`, which is a valid class for `contents`. + unsafe { self.imp.layer.setContents(contents) }; CATransaction::commit(); - Ok(()) } @@ -184,3 +342,17 @@ struct SendCGColorSpace(CGColorSpace); // SAFETY: `CGColorSpace` is immutable, and can freely be shared between threads. unsafe impl Send for SendCGColorSpace {} unsafe impl Sync for SendCGColorSpace {} + +struct SendCALayer(Retained); +// CALayer is thread safe, like most things in Core Animation, see: +// https://developer.apple.com/documentation/quartzcore/catransaction/1448267-lock?language=objc +// https://stackoverflow.com/questions/76250226/how-to-render-content-of-calayer-on-a-background-thread +unsafe impl Send for SendCALayer {} +unsafe impl Sync for SendCALayer {} + +impl Deref for SendCALayer { + type Target = CALayer; + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/src/backends/mod.rs b/src/backends/mod.rs index 8402b441..f700b05a 100644 --- a/src/backends/mod.rs +++ b/src/backends/mod.rs @@ -1,7 +1,7 @@ use crate::{ContextInterface, InitError}; use raw_window_handle::HasDisplayHandle; -#[cfg(target_os = "macos")] +#[cfg(target_vendor = "apple")] pub(crate) mod cg; #[cfg(kms_platform)] pub(crate) mod kms; diff --git a/src/lib.rs b/src/lib.rs index 8b569c79..d19eafa7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -118,7 +118,7 @@ impl Surface { /// ## Platform Dependent Behavior /// /// - On X11, the window must be visible. - /// - On macOS, Redox and Wayland, this function is unimplemented. + /// - On AppKit, UIKit, Redox and Wayland, this function is unimplemented. /// - On Web, this will fail if the content was supplied by /// a different origin depending on the sites CORS rules. pub fn fetch(&mut self) -> Result, SoftBufferError> { @@ -194,7 +194,8 @@ impl HasWindowHandle for Surface /// /// Currently [`Buffer::present`] must block copying image data on: /// - Web -/// - macOS +/// - AppKit +/// - UIKit pub struct Buffer<'a, D, W> { buffer_impl: BufferDispatch<'a, D, W>, _marker: PhantomData<(Arc, Cell<()>)>,