diff --git a/apps/desktop/src/routes/editor/Timeline.tsx b/apps/desktop/src/routes/editor/Timeline.tsx index 8ce4a2cd..56285bf7 100644 --- a/apps/desktop/src/routes/editor/Timeline.tsx +++ b/apps/desktop/src/routes/editor/Timeline.tsx @@ -9,7 +9,10 @@ import { createSignal, onMount, } from "solid-js"; -import { createEventListenerMap } from "@solid-primitives/event-listener"; +import { + createEventListener, + createEventListenerMap, +} from "@solid-primitives/event-listener"; import { cx } from "cva"; import { produce } from "solid-js/store"; import { mergeRefs } from "@solid-primitives/refs"; @@ -88,6 +91,24 @@ export function Timeline() { ); } + let zoomSegmentDragState = { type: "idle" } as + | { type: "idle" } + | { type: "movePending" } + | { type: "moving" }; + + function handleUpdatePlayhead(e: MouseEvent) { + const { left, width } = timelineBounds; + if (zoomSegmentDragState.type !== "moving") + commands.setPlayheadPosition( + videoId, + Math.round( + FPS * + editorInstance.recordingDuration * + ((e.clientX - left!) / width!) + ) + ); + } + return (
{ - const { left, width } = timelineBounds; - commands.setPlayheadPosition( - videoId, - Math.round( - FPS * - editorInstance.recordingDuration * - ((e.clientX - left!) / width!) - ) - ); + createRoot((dispose) => { + createEventListener(e.currentTarget, "mouseup", () => { + handleUpdatePlayhead(e); + setState("timelineSelection", null); + }); + createEventListener(window, "mouseup", () => { + dispose(); + }); + }); }} onMouseMove={(e) => { const { left, width } = timelineBounds; @@ -116,9 +137,6 @@ export function Timeline() { onMouseLeave={() => { setPreviewTime(undefined); }} - onClick={() => { - setState("timelineSelection", null); - }} > {(time) => ( @@ -399,38 +417,48 @@ export function Timeline() { setHoveredTime(Math.min(time, duration() - 1)); }} - onMouseLeave={(e) => setHoveredTime()} - onClick={(e) => { - const time = hoveredTime(); - if (time === undefined) return; - - e.stopPropagation(); - batch(() => { - setProject("timeline", "zoomSegments", (v) => v ?? []); - setProject( - "timeline", - "zoomSegments", - produce((zoomSegments) => { - zoomSegments ??= []; - zoomSegments.push({ - start: time, - end: time + 1, - amount: 1.5, - mode: { - manual: { - x: 0.5, - y: 0.5, - }, - }, - }); - }) - ); + onMouseLeave={() => setHoveredTime()} + onMouseDown={(e) => { + createRoot((dispose) => { + createEventListener(e.currentTarget, "mouseup", (e) => { + dispose(); + + const time = hoveredTime(); + if (time === undefined) return; + + e.stopPropagation(); + batch(() => { + setProject( + "timeline", + "zoomSegments", + (v) => v ?? [] + ); + setProject( + "timeline", + "zoomSegments", + produce((zoomSegments) => { + zoomSegments ??= []; + zoomSegments.push({ + start: time, + end: time + 1, + amount: 1.5, + mode: { + manual: { + x: 0.5, + y: 0.5, + }, + }, + }); + }) + ); + }); + }); }); }} > @@ -451,7 +479,7 @@ export function Timeline() { {(segment, i) => { - const { setTrackState } = useTrackContext(); + const { setTrackState, trackBounds } = useTrackContext(); const zoomPercentage = () => { const amount = segment.amount; @@ -461,6 +489,77 @@ export function Timeline() { const zoomSegments = () => project.timeline!.zoomSegments!; + function handleSegmentMouseUp(e: MouseEvent) {} + + function createMouseDownDrag( + setup: () => T, + _update: ( + e: MouseEvent, + v: T, + initialMouseX: number + ) => void + ) { + return function (downEvent: MouseEvent) { + downEvent.stopPropagation(); + + const initial = setup(); + + let moved = false; + let initialMouseX: null | number = null; + + setTrackState("draggingSegment", true); + + const resumeHistory = history.pause(); + + zoomSegmentDragState = { type: "movePending" }; + + function finish(e: MouseEvent) { + resumeHistory(); + if (!moved) { + e.stopPropagation(); + setState("timelineSelection", { + type: "zoom", + index: i(), + }); + handleUpdatePlayhead(e); + } + zoomSegmentDragState = { type: "idle" }; + setTrackState("draggingSegment", false); + } + + function update(event: MouseEvent) { + if ( + Math.abs(event.clientX - downEvent.clientX) > 2 + ) { + if (!moved) { + moved = true; + initialMouseX = event.clientX; + zoomSegmentDragState = { + type: "moving", + }; + } + } + + if (initialMouseX === null) return; + + _update(event, initial, initialMouseX); + } + + createRoot((dispose) => { + createEventListenerMap(window, { + mousemove: (e) => { + update(e); + }, + mouseup: (e) => { + update(e); + finish(e); + dispose(); + }, + }); + }); + }; + } + return ( { - const start = segment.start; - - let minValue = 0; - - for ( - let i = zoomSegments().length - 1; - i >= 0; - i-- - ) { - const segment = zoomSegments()[i]!; - if (segment.end <= start) { - minValue = segment.end; - break; + onMouseDown={createMouseDownDrag( + () => { + const start = segment.start; + + let minValue = 0; + + let maxValue = segment.end - 1; + + for ( + let i = zoomSegments().length - 1; + i >= 0; + i-- + ) { + const segment = zoomSegments()[i]!; + if (segment.end <= start) { + minValue = segment.end; + break; + } } - } - let maxValue = segment.end - 1; - - function update(event: MouseEvent) { + return { start, minValue, maxValue }; + }, + (e, value, initialMouseX) => { const { width } = timelineBounds; const newStart = - start + - ((event.clientX - downEvent.clientX) / - width!) * + value.start + + ((e.clientX - initialMouseX) / width!) * duration(); setProject( @@ -509,40 +610,57 @@ export function Timeline() { i(), "start", Math.min( - maxValue, - Math.max(minValue, newStart) + value.maxValue, + Math.max(value.minValue, newStart) ) ); } - - setTrackState("draggingHandle", true); - - const resumeHistory = history.pause(); - createRoot((dispose) => { - createEventListenerMap(window, { - mousemove: update, - mouseup: (e) => { - dispose(); - resumeHistory(); - update(e); - setTrackState("draggingHandle", false); - }, - }); - }); - }} - onClick={(e) => { - e.stopPropagation(); - }} + )} /> { - setState("timelineSelection", { - type: "zoom", - index: i(), - }); - e.stopPropagation(); - }} + onMouseDown={createMouseDownDrag( + () => { + const original = { ...segment }; + + const prevSegment = zoomSegments()[i() - 1]; + const nextSegment = zoomSegments()[i() + 1]; + + const minStart = prevSegment?.end ?? 0; + const maxEnd = nextSegment?.start ?? duration(); + + const secsPerPixel = + duration() / trackBounds.width!; + + return { + secsPerPixel, + original, + minStart, + maxEnd, + }; + }, + (e, value, initialMouseX) => { + const rawDelta = + (e.clientX - initialMouseX) * + value.secsPerPixel; + + const newStart = + value.original.start + rawDelta; + const newEnd = value.original.end + rawDelta; + + let delta = rawDelta; + + if (newStart < value.minStart) + delta = value.minStart - value.original.start; + else if (newEnd > value.maxEnd) + delta = value.maxEnd - value.original.end; + + setProject("timeline", "zoomSegments", i(), { + start: value.original.start + delta, + end: value.original.end + delta, + }); + } + )} > {(() => { const ctx = useSegmentContext(); @@ -561,40 +679,34 @@ export function Timeline() { { - const end = segment.end; - - const minValue = segment.start + 1; - - let maxValue = duration(); - - for (let i = 0; i > zoomSegments().length; i++) { - const segment = zoomSegments()[i]!; - if (segment.start > end) { - maxValue = segment.end; - break; + onMouseDown={createMouseDownDrag( + () => { + const end = segment.end; + + const minValue = segment.start + 1; + + let maxValue = duration(); + + for ( + let i = 0; + i < zoomSegments().length; + i++ + ) { + const segment = zoomSegments()[i]!; + if (segment.start > end) { + maxValue = segment.start; + break; + } } - } - const maxDuration = - editorInstance.recordingDuration - - segments().reduce( - (acc, segment, segmentI) => - segmentI === i() - ? acc - : acc + - (segment.end - segment.start) / - segment.timescale, - 0 - ); - - function update(event: MouseEvent) { + return { end, minValue, maxValue }; + }, + (e, value, initialMouseX) => { const { width } = timelineBounds; const newEnd = - end + - ((event.clientX - downEvent.clientX) / - width!) * + value.end + + ((e.clientX - initialMouseX) / width!) * duration(); setProject( @@ -602,28 +714,13 @@ export function Timeline() { "zoomSegments", i(), "end", - Math.min(maxValue, Math.max(minValue, newEnd)) + Math.min( + value.maxValue, + Math.max(value.minValue, newEnd) + ) ); } - - setTrackState("draggingHandle", true); - - const resumeHistory = history.pause(); - createRoot((dispose) => { - createEventListenerMap(window, { - mousemove: update, - mouseup: (e) => { - dispose(); - resumeHistory(); - update(e); - setTrackState("draggingHandle", false); - }, - }); - }); - }} - onClick={(e) => { - e.stopPropagation(); - }} + )} /> ); @@ -659,11 +756,14 @@ function SegmentRoot( props: ComponentProps<"div"> & { innerClass: string; segment: { start: number; end: number }; + onMouseDown?: ( + e: MouseEvent & { currentTarget: HTMLDivElement; target: Element } + ) => void; } ) { const { duration } = useTimelineContext(); - const { trackBounds, isFreeForm } = useTrackContext(); - const { state, project } = useEditorContext(); + const { trackBounds, isFreeForm, setTrackState } = useTrackContext(); + const { state, project, history, setProject } = useEditorContext(); const isSelected = createMemo(() => { const selection = state.timelineSelection; @@ -700,7 +800,6 @@ function SegmentRoot( transform: isFreeForm() ? "translateX(var(--segment-x))" : undefined, width: `${width()}px`, }} - onMouseDown={props.onMouseDown} ref={props.ref} >
; }) => { const [trackState, setTrackState] = createStore({ - draggingHandle: false, + draggingSegment: false, }); const bounds = createElementBounds(() => props.ref()); diff --git a/crates/editor/src/editor.rs b/crates/editor/src/editor.rs index 9c444a09..a6392956 100644 --- a/crates/editor/src/editor.rs +++ b/crates/editor/src/editor.rs @@ -1,7 +1,7 @@ use std::{sync::Arc, time::Instant}; use cap_media::{feeds::RawCameraFrame, frame_ws::WSFrame}; -use cap_project::{BackgroundSource, ProjectConfiguration, RecordingMeta, XY}; +use cap_project::{BackgroundSource, RecordingMeta, XY}; use cap_rendering::{ decoder::DecodedFrame, produce_frame, ProjectRecordings, ProjectUniforms, RenderVideoConstants, }; diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index 9891f78c..2f0325be 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -51,7 +51,7 @@ impl Default for BackgroundSource { } } -#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] +#[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default, PartialEq)] #[serde(rename_all = "camelCase")] pub struct XY { pub x: T, diff --git a/crates/rendering/src/decoder.rs b/crates/rendering/src/decoder.rs deleted file mode 100644 index ddc37606..00000000 --- a/crates/rendering/src/decoder.rs +++ /dev/null @@ -1,839 +0,0 @@ -use std::{ - cell::LazyCell, - collections::BTreeMap, - path::PathBuf, - sync::{mpsc, Arc}, -}; - -use cidre::cv::pixel_buffer::LockFlags; -use ffmpeg::{ - codec::{self, Capabilities}, - format, frame, rescale, software, Codec, Rational, Rescale, -}; -use ffmpeg_hw_device::{CodecContextExt, HwDevice}; -use ffmpeg_sys_next::{avcodec_find_decoder, AVHWDeviceType}; - -pub type DecodedFrame = Arc>; - -pub enum VideoDecoderMessage { - GetFrame(f32, tokio::sync::oneshot::Sender), -} - -fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 { - (fps as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))) - .round() as u32 -} - -const FRAME_CACHE_SIZE: usize = 100; - -#[derive(Clone)] -struct CachedFrame { - data: CachedFrameData, -} - -impl CachedFrame { - fn process(&mut self, decoder: &codec::decoder::Video) -> Arc> { - match &mut self.data { - CachedFrameData::Raw(frame) => { - let rgb_frame = if frame.format() != format::Pixel::RGBA { - // Reinitialize the scaler with the new input format - let mut scaler = software::converter( - (decoder.width(), decoder.height()), - frame.format(), - format::Pixel::RGBA, - ) - .unwrap(); - - let mut rgb_frame = frame::Video::empty(); - scaler.run(&frame, &mut rgb_frame).unwrap(); - rgb_frame - } else { - std::mem::replace(frame, frame::Video::empty()) - }; - - let width = rgb_frame.width() as usize; - let height = rgb_frame.height() as usize; - let stride = rgb_frame.stride(0); - let data = rgb_frame.data(0); - - let expected_size = width * height * 4; - - let mut frame_buffer = Vec::with_capacity(expected_size); - - // account for stride > width - for line_data in data.chunks_exact(stride) { - frame_buffer.extend_from_slice(&line_data[0..width * 4]); - } - - let data = Arc::new(frame_buffer); - - self.data = CachedFrameData::Processed(data.clone()); - - data - } - CachedFrameData::Processed(data) => data.clone(), - } - } -} - -#[derive(Clone)] -enum CachedFrameData { - Raw(frame::Video), - Processed(Arc>), -} - -pub struct AsyncVideoDecoder; - -#[derive(Clone)] -pub struct AsyncVideoDecoderHandle { - sender: mpsc::Sender, -} - -impl AsyncVideoDecoderHandle { - pub async fn get_frame(&self, time: f32) -> Option { - let (tx, rx) = tokio::sync::oneshot::channel(); - self.sender - .send(VideoDecoderMessage::GetFrame(time, tx)) - .unwrap(); - rx.await.ok() - } -} - -pub enum GetFrameError { - Failed, -} - -struct PeekableReceiver { - rx: mpsc::Receiver, - peeked: Option, -} - -impl PeekableReceiver { - fn peek(&mut self) -> Option<&T> { - if self.peeked.is_some() { - self.peeked.as_ref() - } else { - match self.rx.try_recv() { - Ok(value) => { - self.peeked = Some(value); - self.peeked.as_ref() - } - Err(_) => None, - } - } - } - - fn recv(&mut self) -> Result { - if let Some(value) = self.peeked.take() { - Ok(value) - } else { - self.rx.recv() - } - } -} - -fn ff_find_decoder( - s: &format::context::Input, - st: &format::stream::Stream, - codec_id: codec::Id, -) -> Option { - unsafe { - use ffmpeg::media::Type; - let codec = match st.parameters().medium() { - Type::Video => Some((*s.as_ptr()).video_codec), - Type::Audio => Some((*s.as_ptr()).audio_codec), - Type::Subtitle => Some((*s.as_ptr()).subtitle_codec), - _ => None, - }; - - if let Some(codec) = codec { - if !codec.is_null() { - return Some(Codec::wrap(codec)); - } - } - - let found = avcodec_find_decoder(codec_id.into()); - - if found.is_null() { - return None; - } - Some(Codec::wrap(found)) - } -} - -pub enum Decoder { - Ffmpeg(FfmpegDecoder), - #[cfg(target_os = "macos")] - AVAssetReader(AVAssetReaderDecoder), -} - -impl Decoder { - pub fn spawn(name: &'static str, path: PathBuf, fps: u32) -> AsyncVideoDecoderHandle { - let (tx, rx) = mpsc::channel(); - - let handle = AsyncVideoDecoderHandle { sender: tx }; - - #[cfg(target_os = "macos")] - { - AVAssetReaderDecoder::spawn(name, path, fps, rx); - } - - #[cfg(not(target_os = "macos"))] - { - FfmpegDecoder::spawn(name, path, fps, rx); - } - - handle - } -} - -struct FfmpegDecoder; - -impl FfmpegDecoder { - fn spawn(name: &'static str, path: PathBuf, fps: u32, rx: mpsc::Receiver) { - std::thread::spawn(move || { - let mut input = ffmpeg::format::input(&path).unwrap(); - - let input_stream = input - .streams() - .best(ffmpeg::media::Type::Video) - .ok_or("Could not find a video stream") - .unwrap(); - - let decoder_codec = - ff_find_decoder(&input, &input_stream, input_stream.parameters().id()).unwrap(); - - let mut context = codec::context::Context::new_with_codec(decoder_codec); - context.set_parameters(input_stream.parameters()).unwrap(); - - let input_stream_index = input_stream.index(); - let time_base = input_stream.time_base(); - let frame_rate = input_stream.rate(); - - // Create a decoder for the video stream - let mut decoder = context.decoder().video().unwrap(); - - { - use codec::threading::{Config, Type}; - - let capabilities = decoder_codec.capabilities(); - - if capabilities.intersects(Capabilities::FRAME_THREADS) { - decoder.set_threading(Config::kind(Type::Frame)); - } else if capabilities.intersects(Capabilities::SLICE_THREADS) { - decoder.set_threading(Config::kind(Type::Slice)); - } else { - decoder.set_threading(Config::count(1)); - } - } - - let hw_device: Option = { - #[cfg(target_os = "macos")] - { - decoder - .try_use_hw_device( - AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX, - Pixel::NV12, - ) - .ok() - } - #[cfg(target_os = "windows")] - { - decoder - .try_use_hw_device(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA, Pixel::NV12) - .ok() - } - - #[cfg(not(any(target_os = "macos", target_os = "windows")))] - None - }; - - use ffmpeg::format::Pixel; - - let mut temp_frame = ffmpeg::frame::Video::empty(); - - // let mut packets = input.packets().peekable(); - - let width = decoder.width(); - let height = decoder.height(); - let black_frame = LazyCell::new((|| Arc::new(vec![0; (width * height * 4) as usize]))); - - let mut cache = BTreeMap::::new(); - // active frame is a frame that triggered decode. - // frames that are within render_more_margin of this frame won't trigger decode. - let mut last_active_frame = None::; - - let mut last_decoded_frame = None::; - let mut last_sent_frame = None::<(u32, DecodedFrame)>; - - let mut peekable_requests = PeekableReceiver { rx, peeked: None }; - - let mut packets = input.packets().peekable(); - - while let Ok(r) = peekable_requests.recv() { - match r { - VideoDecoderMessage::GetFrame(requested_time, sender) => { - let requested_frame = (requested_time * fps as f32).floor() as u32; - // sender.send(black_frame.clone()).ok(); - // continue; - - let mut sender = if let Some(cached) = cache.get_mut(&requested_frame) { - let data = cached.process(&decoder); - - sender.send(data.clone()).ok(); - last_sent_frame = Some((requested_frame, data)); - continue; - } else { - Some(sender) - }; - - let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; - - if requested_frame <= 0 - || last_sent_frame - .as_ref() - .map(|last| { - requested_frame < last.0 || - // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future - requested_frame - last.0 > FRAME_CACHE_SIZE as u32 - }) - .unwrap_or(true) - { - let timestamp_us = - ((requested_frame as f32 / frame_rate.numerator() as f32) - * 1_000_000.0) as i64; - let position = timestamp_us.rescale((1, 1_000_000), rescale::TIME_BASE); - - decoder.flush(); - input.seek(position, ..position).unwrap(); - last_decoded_frame = None; - - packets = input.packets().peekable(); - } - - last_active_frame = Some(requested_frame); - - loop { - if peekable_requests.peek().is_some() { - break; - } - let Some((stream, packet)) = packets.next() else { - // handles the case where the cache doesn't contain a frame so we fallback to the previously sent one - if let Some(last_sent_frame) = &last_sent_frame { - if last_sent_frame.0 < requested_frame { - sender.take().map(|s| s.send(last_sent_frame.1.clone())); - } - } - - sender.take().map(|s| s.send(black_frame.clone())); - break; - }; - - if stream.index() == input_stream_index { - let start_offset = stream.start_time(); - - decoder.send_packet(&packet).ok(); // decode failures are ok, we just fail to return a frame - - let mut exit = false; - - while decoder.receive_frame(&mut temp_frame).is_ok() { - let current_frame = pts_to_frame( - temp_frame.pts().unwrap() - start_offset, - time_base, - fps, - ); - - // Handles frame skips. requested_frame == last_decoded_frame should be handled by the frame cache. - if let Some((last_decoded_frame, sender)) = last_decoded_frame - .filter(|last_decoded_frame| { - requested_frame > *last_decoded_frame - && requested_frame < current_frame - }) - .and_then(|l| Some((l, sender.take()?))) - { - let data = cache - .get_mut(&last_decoded_frame) - .map(|f| f.process(&decoder)) - .unwrap_or_else(|| black_frame.clone()); - - last_sent_frame = Some((last_decoded_frame, data.clone())); - sender.send(data).ok(); - } - - last_decoded_frame = Some(current_frame); - - let exceeds_cache_bounds = current_frame > cache_max; - let too_small_for_cache_bounds = current_frame < cache_min; - - let hw_frame = - hw_device.as_ref().and_then(|d| d.get_hwframe(&temp_frame)); - - let frame = hw_frame.unwrap_or(std::mem::replace( - &mut temp_frame, - frame::Video::empty(), - )); - - if !too_small_for_cache_bounds { - let mut cache_frame = CachedFrame { - data: CachedFrameData::Raw(frame), - }; - - if current_frame == requested_frame { - if let Some(sender) = sender.take() { - let data = cache_frame.process(&decoder); - last_sent_frame = - Some((current_frame, data.clone())); - sender.send(data).ok(); - - break; - } - } - - if cache.len() >= FRAME_CACHE_SIZE { - if let Some(last_active_frame) = &last_active_frame { - let frame = if requested_frame > *last_active_frame - { - *cache.keys().next().unwrap() - } else if requested_frame < *last_active_frame { - *cache.keys().next_back().unwrap() - } else { - let min = *cache.keys().min().unwrap(); - let max = *cache.keys().max().unwrap(); - - if current_frame > max { - min - } else { - max - } - }; - - cache.remove(&frame); - } else { - cache.clear() - } - } - - cache.insert(current_frame, cache_frame); - } - - exit = exit || exceeds_cache_bounds; - } - - if exit { - break; - } - } - } - - if let Some((sender, last_sent_frame)) = - sender.take().zip(last_sent_frame.clone()) - { - sender.send(last_sent_frame.1).ok(); - } - } - } - } - }); - } -} - -use tokio::runtime::Handle as TokioHandle; - -#[cfg(target_os = "macos")] -mod macos { - use super::*; - use cidre::{arc::R, av, cm, cv, ns}; - use format::context::input::PacketIter; - - #[derive(Clone)] - enum CachedFrame { - Raw(R), - Processed(Arc>), - } - - impl CachedFrame { - fn process(&mut self) -> Arc> { - match self { - CachedFrame::Raw(image_buf) => { - let format = pixel_format_to_pixel(image_buf.pixel_format()); - - let data = if matches!(format, format::Pixel::RGBA) { - let _lock = image_buf.base_address_lock(LockFlags::READ_ONLY).unwrap(); - - let bytes_per_row = image_buf.plane_bytes_per_row(0); - let width = image_buf.width() as usize; - let height = image_buf.height(); - - let slice = unsafe { - std::slice::from_raw_parts::<'static, _>( - image_buf.plane_base_address(0), - bytes_per_row * height, - ) - }; - - let mut bytes = Vec::with_capacity(width * height * 4); - - let row_length = width * 4; - - for i in 0..height { - bytes.as_mut_slice()[i * row_length..((i + 1) * row_length)] - .copy_from_slice( - &slice[i * bytes_per_row..(i * bytes_per_row + row_length)], - ) - } - - bytes - } else { - let mut ffmpeg_frame = ffmpeg::frame::Video::new( - format, - image_buf.width() as u32, - image_buf.height() as u32, - ); - - match ffmpeg_frame.format() { - format::Pixel::NV12 => { - let _lock = - image_buf.base_address_lock(LockFlags::READ_ONLY).unwrap(); - - for plane_i in 0..image_buf.plane_count() { - let bytes_per_row = image_buf.plane_bytes_per_row(plane_i); - let height = image_buf.plane_height(plane_i); - - let ffmpeg_stride = ffmpeg_frame.stride(plane_i); - let row_length = bytes_per_row.min(ffmpeg_stride); - - let slice = unsafe { - std::slice::from_raw_parts::<'static, _>( - image_buf.plane_base_address(plane_i), - bytes_per_row * height, - ) - }; - - for i in 0..height { - ffmpeg_frame.data_mut(plane_i) - [i * ffmpeg_stride..(i * ffmpeg_stride + row_length)] - .copy_from_slice( - &slice[i * bytes_per_row - ..(i * bytes_per_row + row_length)], - ) - } - } - } - format::Pixel::YUV420P => { - let _lock = - image_buf.base_address_lock(LockFlags::READ_ONLY).unwrap(); - - for plane_i in 0..image_buf.plane_count() { - let bytes_per_row = image_buf.plane_bytes_per_row(plane_i); - let height = image_buf.plane_height(plane_i); - - let ffmpeg_stride = ffmpeg_frame.stride(plane_i); - let row_length = bytes_per_row.min(ffmpeg_stride); - - let slice = unsafe { - std::slice::from_raw_parts::<'static, _>( - image_buf.plane_base_address(plane_i), - bytes_per_row * height, - ) - }; - - for i in 0..height { - ffmpeg_frame.data_mut(plane_i) - [i * ffmpeg_stride..(i * ffmpeg_stride + row_length)] - .copy_from_slice( - &slice[i * bytes_per_row - ..(i * bytes_per_row + row_length)], - ) - } - } - } - format => todo!("implement {:?}", format), - } - - let mut converter = ffmpeg::software::converter( - (ffmpeg_frame.width(), ffmpeg_frame.height()), - ffmpeg_frame.format(), - format::Pixel::RGBA, - ) - .unwrap(); - - let mut rgb_frame = frame::Video::empty(); - converter.run(&ffmpeg_frame, &mut rgb_frame).unwrap(); - - rgb_frame.data(0).to_vec() - }; - - let data = Arc::new(data); - - *self = Self::Processed(data.clone()); - - data - } - CachedFrame::Processed(data) => data.clone(), - } - } - } - - pub struct AVAssetReaderDecoder; - - impl AVAssetReaderDecoder { - pub fn spawn( - name: &'static str, - path: PathBuf, - fps: u32, - rx: mpsc::Receiver, - ) { - let handle = tokio::runtime::Handle::current(); - - std::thread::spawn(move || { - let (pixel_format, width, height) = { - let input = ffmpeg::format::input(&path).unwrap(); - - let input_stream = input - .streams() - .best(ffmpeg::media::Type::Video) - .ok_or("Could not find a video stream") - .unwrap(); - - let decoder_codec = - ff_find_decoder(&input, &input_stream, input_stream.parameters().id()) - .unwrap(); - - let mut context = codec::context::Context::new_with_codec(decoder_codec); - context.set_parameters(input_stream.parameters()).unwrap(); - - let decoder = context.decoder().video().unwrap(); - ( - pixel_to_pixel_format(decoder.format()), - decoder.width(), - decoder.height(), - ) - }; - - let asset = av::UrlAsset::with_url( - &ns::Url::with_fs_path_str(path.to_str().unwrap(), false), - None, - ) - .unwrap(); - - fn get_reader_track_output( - asset: &av::UrlAsset, - time: f32, - handle: &TokioHandle, - pixel_format: cv::PixelFormat, - ) -> R { - let mut reader = av::AssetReader::with_asset(&asset).unwrap(); - - let time_range = cm::TimeRange { - start: cm::Time::with_secs(time as f64, 100), - duration: asset.duration(), - }; - - reader.set_time_range(time_range); - - let tracks = handle - .block_on(asset.load_tracks_with_media_type(av::MediaType::video())) - .unwrap(); - - let track = tracks.get(0).unwrap(); - - let mut reader_track_output = av::AssetReaderTrackOutput::with_track( - &track, - Some(&ns::Dictionary::with_keys_values( - &[cv::pixel_buffer::keys::pixel_format().as_ns()], - &[pixel_format.to_cf_number().as_ns().as_id_ref()], - )), - ) - .unwrap(); - - reader_track_output.set_always_copies_sample_data(false); - - reader.add_output(&reader_track_output).unwrap(); - - reader.start_reading().ok(); - - reader_track_output - } - - let mut track_output = get_reader_track_output(&asset, 0.0, &handle, pixel_format); - - let black_frame = - LazyCell::new(|| Arc::new(vec![0; (width * height * 4) as usize])); - - let mut cache = BTreeMap::::new(); - - let mut last_active_frame = None::; - - let mut last_decoded_frame = None::; - let mut last_sent_frame = None::<(u32, DecodedFrame)>; - - while let Ok(r) = rx.recv() { - match r { - VideoDecoderMessage::GetFrame(requested_time, sender) => { - let requested_frame = (requested_time * fps as f32).floor() as u32; - - let mut sender = if let Some(cached) = cache.get_mut(&requested_frame) { - let data = cached.process(); - - sender.send(data.clone()).ok(); - last_sent_frame = Some((requested_frame, data)); - continue; - } else { - Some(sender) - }; - - let cache_min = - requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); - let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; - - if requested_frame <= 0 - || last_sent_frame - .as_ref() - .map(|last| { - requested_frame < last.0 || - // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future - requested_frame - last.0 > FRAME_CACHE_SIZE as u32 - }) - .unwrap_or(true) - { - track_output = get_reader_track_output( - &asset, - requested_time, - &handle, - pixel_format, - ); - last_decoded_frame = None; - } - - last_active_frame = Some(requested_frame); - - let mut exit = false; - - while let Some((current_frame, mut cache_frame)) = track_output - .next_sample_buf() - .unwrap() - .and_then(|sample_buf| { - let current_frame = pts_to_frame( - sample_buf.pts().value, - Rational::new(1, sample_buf.pts().scale), - fps, - ); - - let image_buf = sample_buf.image_buf()?; - - Some((current_frame, CachedFrame::Raw(image_buf.retained()))) - }) - { - // Handles frame skips. requested_frame == last_decoded_frame should be handled by the frame cache. - if let Some((last_decoded_frame, sender)) = last_decoded_frame - .filter(|last_decoded_frame| { - requested_frame > *last_decoded_frame - && requested_frame < current_frame - }) - .and_then(|l| Some((l, sender.take()?))) - { - let data = cache - .get_mut(&last_decoded_frame) - .map(|f| f.process()) - .unwrap_or_else(|| black_frame.clone()); - - last_sent_frame = Some((last_decoded_frame, data.clone())); - sender.send(data).ok(); - } - - last_decoded_frame = Some(current_frame); - - let exceeds_cache_bounds = current_frame > cache_max; - let too_small_for_cache_bounds = current_frame < cache_min; - - if !too_small_for_cache_bounds { - if current_frame == requested_frame { - if let Some(sender) = sender.take() { - let data = cache_frame.process(); - last_sent_frame = Some((current_frame, data.clone())); - sender.send(data).ok(); - - break; - } - } - - if cache.len() >= FRAME_CACHE_SIZE { - if let Some(last_active_frame) = &last_active_frame { - let frame = if requested_frame > *last_active_frame { - *cache.keys().next().unwrap() - } else if requested_frame < *last_active_frame { - *cache.keys().next_back().unwrap() - } else { - let min = *cache.keys().min().unwrap(); - let max = *cache.keys().max().unwrap(); - - if current_frame > max { - min - } else { - max - } - }; - - cache.remove(&frame); - } else { - cache.clear() - } - } - - cache.insert(current_frame, cache_frame.clone()); - } - - if current_frame > requested_frame && sender.is_some() { - if let Some((sender, last_sent_frame)) = last_sent_frame - .as_ref() - .and_then(|l| Some((sender.take()?, l))) - { - sender.send(last_sent_frame.1.clone()).ok(); - } else if let Some(sender) = sender.take() { - sender.send(cache_frame.process()).ok(); - } - } - - exit = exit || exceeds_cache_bounds; - - if exit { - break; - } - } - - if let Some((sender, last_sent_frame)) = - sender.take().zip(last_sent_frame.as_ref()) - { - sender.send(last_sent_frame.1.clone()).ok(); - } - } - } - } - - println!("Decoder thread ended"); - }); - } - } - - fn pixel_to_pixel_format(pixel: format::Pixel) -> cv::PixelFormat { - match pixel { - format::Pixel::NV12 => cv::PixelFormat::_420V, - // this is intentional, it works and is faster /shrug - format::Pixel::YUV420P => cv::PixelFormat::_420V, - format::Pixel::RGBA => cv::PixelFormat::_32_RGBA, - _ => todo!(), - } - } - - fn pixel_format_to_pixel(format: cv::PixelFormat) -> format::Pixel { - match format { - cv::PixelFormat::_420V => format::Pixel::NV12, - cv::PixelFormat::_32_RGBA => format::Pixel::RGBA, - _ => todo!(), - } - } -} - -#[cfg(target_os = "macos")] -use macos::*; diff --git a/crates/rendering/src/decoder/avassetreader.rs b/crates/rendering/src/decoder/avassetreader.rs new file mode 100644 index 00000000..401e3dc4 --- /dev/null +++ b/crates/rendering/src/decoder/avassetreader.rs @@ -0,0 +1,404 @@ +use std::{ + cell::LazyCell, + collections::BTreeMap, + path::PathBuf, + sync::{mpsc, Arc}, +}; + +use cidre::{ + arc::R, + av, cm, + cv::{self, pixel_buffer::LockFlags}, + ns, +}; +use ffmpeg::{codec, format, frame, Rational}; +use tokio::runtime::Handle as TokioHandle; + +use super::{pts_to_frame, DecodedFrame, VideoDecoderMessage, FRAME_CACHE_SIZE}; + +#[derive(Clone)] +enum CachedFrame { + Raw(R), + Processed(Arc>), +} + +impl CachedFrame { + fn process(&mut self) -> Arc> { + match self { + CachedFrame::Raw(image_buf) => { + let format = pixel_format_to_pixel(image_buf.pixel_format()); + + let data = if matches!(format, format::Pixel::RGBA) { + let _lock = image_buf.base_address_lock(LockFlags::READ_ONLY).unwrap(); + + let bytes_per_row = image_buf.plane_bytes_per_row(0); + let width = image_buf.width() as usize; + let height = image_buf.height(); + + let slice = unsafe { + std::slice::from_raw_parts::<'static, _>( + image_buf.plane_base_address(0), + bytes_per_row * height, + ) + }; + + let mut bytes = Vec::with_capacity(width * height * 4); + + let row_length = width * 4; + + for i in 0..height { + bytes.as_mut_slice()[i * row_length..((i + 1) * row_length)] + .copy_from_slice( + &slice[i * bytes_per_row..(i * bytes_per_row + row_length)], + ) + } + + bytes + } else { + let mut ffmpeg_frame = ffmpeg::frame::Video::new( + format, + image_buf.width() as u32, + image_buf.height() as u32, + ); + + match ffmpeg_frame.format() { + format::Pixel::NV12 => { + let _lock = image_buf.base_address_lock(LockFlags::READ_ONLY).unwrap(); + + for plane_i in 0..image_buf.plane_count() { + let bytes_per_row = image_buf.plane_bytes_per_row(plane_i); + let height = image_buf.plane_height(plane_i); + + let ffmpeg_stride = ffmpeg_frame.stride(plane_i); + let row_length = bytes_per_row.min(ffmpeg_stride); + + let slice = unsafe { + std::slice::from_raw_parts::<'static, _>( + image_buf.plane_base_address(plane_i), + bytes_per_row * height, + ) + }; + + for i in 0..height { + ffmpeg_frame.data_mut(plane_i) + [i * ffmpeg_stride..(i * ffmpeg_stride + row_length)] + .copy_from_slice( + &slice[i * bytes_per_row + ..(i * bytes_per_row + row_length)], + ) + } + } + } + format::Pixel::YUV420P => { + let _lock = image_buf.base_address_lock(LockFlags::READ_ONLY).unwrap(); + + for plane_i in 0..image_buf.plane_count() { + let bytes_per_row = image_buf.plane_bytes_per_row(plane_i); + let height = image_buf.plane_height(plane_i); + + let ffmpeg_stride = ffmpeg_frame.stride(plane_i); + let row_length = bytes_per_row.min(ffmpeg_stride); + + let slice = unsafe { + std::slice::from_raw_parts::<'static, _>( + image_buf.plane_base_address(plane_i), + bytes_per_row * height, + ) + }; + + for i in 0..height { + ffmpeg_frame.data_mut(plane_i) + [i * ffmpeg_stride..(i * ffmpeg_stride + row_length)] + .copy_from_slice( + &slice[i * bytes_per_row + ..(i * bytes_per_row + row_length)], + ) + } + } + } + format => todo!("implement {:?}", format), + } + + let mut converter = ffmpeg::software::converter( + (ffmpeg_frame.width(), ffmpeg_frame.height()), + ffmpeg_frame.format(), + format::Pixel::RGBA, + ) + .unwrap(); + + let mut rgb_frame = frame::Video::empty(); + converter.run(&ffmpeg_frame, &mut rgb_frame).unwrap(); + + rgb_frame.data(0).to_vec() + }; + + let data = Arc::new(data); + + *self = Self::Processed(data.clone()); + + data + } + CachedFrame::Processed(data) => data.clone(), + } + } +} + +pub struct AVAssetReaderDecoder; + +impl AVAssetReaderDecoder { + pub fn spawn( + name: &'static str, + path: PathBuf, + fps: u32, + rx: mpsc::Receiver, + ) { + let handle = tokio::runtime::Handle::current(); + + std::thread::spawn(move || { + let (pixel_format, width, height) = { + let input = ffmpeg::format::input(&path).unwrap(); + + let input_stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or("Could not find a video stream") + .unwrap(); + + let decoder_codec = super::ffmpeg::find_decoder( + &input, + &input_stream, + input_stream.parameters().id(), + ) + .unwrap(); + + let mut context = codec::context::Context::new_with_codec(decoder_codec); + context.set_parameters(input_stream.parameters()).unwrap(); + + let decoder = context.decoder().video().unwrap(); + ( + pixel_to_pixel_format(decoder.format()), + decoder.width(), + decoder.height(), + ) + }; + + let asset = av::UrlAsset::with_url( + &ns::Url::with_fs_path_str(path.to_str().unwrap(), false), + None, + ) + .unwrap(); + + fn get_reader_track_output( + asset: &av::UrlAsset, + time: f32, + handle: &TokioHandle, + pixel_format: cv::PixelFormat, + ) -> R { + let mut reader = av::AssetReader::with_asset(&asset).unwrap(); + + let time_range = cm::TimeRange { + start: cm::Time::with_secs(time as f64, 100), + duration: asset.duration(), + }; + + reader.set_time_range(time_range); + + let tracks = handle + .block_on(asset.load_tracks_with_media_type(av::MediaType::video())) + .unwrap(); + + let track = tracks.get(0).unwrap(); + + let mut reader_track_output = av::AssetReaderTrackOutput::with_track( + &track, + Some(&ns::Dictionary::with_keys_values( + &[cv::pixel_buffer::keys::pixel_format().as_ns()], + &[pixel_format.to_cf_number().as_ns().as_id_ref()], + )), + ) + .unwrap(); + + reader_track_output.set_always_copies_sample_data(false); + + reader.add_output(&reader_track_output).unwrap(); + + reader.start_reading().ok(); + + reader_track_output + } + + let mut track_output = get_reader_track_output(&asset, 0.0, &handle, pixel_format); + + let black_frame = LazyCell::new(|| Arc::new(vec![0; (width * height * 4) as usize])); + + let mut cache = BTreeMap::::new(); + + let mut last_active_frame = None::; + + let mut last_decoded_frame = None::; + let mut last_sent_frame = None::<(u32, DecodedFrame)>; + + while let Ok(r) = rx.recv() { + match r { + VideoDecoderMessage::GetFrame(requested_time, sender) => { + let requested_frame = (requested_time * fps as f32).floor() as u32; + + let mut sender = if let Some(cached) = cache.get_mut(&requested_frame) { + let data = cached.process(); + + sender.send(data.clone()).ok(); + last_sent_frame = Some((requested_frame, data)); + continue; + } else { + Some(sender) + }; + + let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; + + if requested_frame <= 0 + || last_sent_frame + .as_ref() + .map(|last| { + requested_frame < last.0 || + // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future + requested_frame - last.0 > FRAME_CACHE_SIZE as u32 + }) + .unwrap_or(true) + { + track_output = get_reader_track_output( + &asset, + requested_time, + &handle, + pixel_format, + ); + last_decoded_frame = None; + } + + last_active_frame = Some(requested_frame); + + let mut exit = false; + + while let Some((current_frame, mut cache_frame)) = track_output + .next_sample_buf() + .unwrap() + .and_then(|sample_buf| { + let current_frame = pts_to_frame( + sample_buf.pts().value, + Rational::new(1, sample_buf.pts().scale), + fps, + ); + + let image_buf = sample_buf.image_buf()?; + + Some((current_frame, CachedFrame::Raw(image_buf.retained()))) + }) + { + // Handles frame skips. requested_frame == last_decoded_frame should be handled by the frame cache. + if let Some((last_decoded_frame, sender)) = last_decoded_frame + .filter(|last_decoded_frame| { + requested_frame > *last_decoded_frame + && requested_frame < current_frame + }) + .and_then(|l| Some((l, sender.take()?))) + { + let data = cache + .get_mut(&last_decoded_frame) + .map(|f| f.process()) + .unwrap_or_else(|| black_frame.clone()); + + last_sent_frame = Some((last_decoded_frame, data.clone())); + sender.send(data).ok(); + } + + last_decoded_frame = Some(current_frame); + + let exceeds_cache_bounds = current_frame > cache_max; + let too_small_for_cache_bounds = current_frame < cache_min; + + if !too_small_for_cache_bounds { + if current_frame == requested_frame { + if let Some(sender) = sender.take() { + let data = cache_frame.process(); + last_sent_frame = Some((current_frame, data.clone())); + sender.send(data).ok(); + + break; + } + } + + if cache.len() >= FRAME_CACHE_SIZE { + if let Some(last_active_frame) = &last_active_frame { + let frame = if requested_frame > *last_active_frame { + *cache.keys().next().unwrap() + } else if requested_frame < *last_active_frame { + *cache.keys().next_back().unwrap() + } else { + let min = *cache.keys().min().unwrap(); + let max = *cache.keys().max().unwrap(); + + if current_frame > max { + min + } else { + max + } + }; + + cache.remove(&frame); + } else { + cache.clear() + } + } + + cache.insert(current_frame, cache_frame.clone()); + } + + if current_frame > requested_frame && sender.is_some() { + if let Some((sender, last_sent_frame)) = last_sent_frame + .as_ref() + .and_then(|l| Some((sender.take()?, l))) + { + sender.send(last_sent_frame.1.clone()).ok(); + } else if let Some(sender) = sender.take() { + sender.send(cache_frame.process()).ok(); + } + } + + exit = exit || exceeds_cache_bounds; + + if exit { + break; + } + } + + if let Some((sender, last_sent_frame)) = + sender.take().zip(last_sent_frame.as_ref()) + { + sender.send(last_sent_frame.1.clone()).ok(); + } + } + } + } + + println!("Decoder thread ended"); + }); + } +} + +fn pixel_to_pixel_format(pixel: format::Pixel) -> cv::PixelFormat { + match pixel { + format::Pixel::NV12 => cv::PixelFormat::_420V, + // this is intentional, it works and is faster /shrug + format::Pixel::YUV420P => cv::PixelFormat::_420V, + format::Pixel::RGBA => cv::PixelFormat::_32_RGBA, + _ => todo!(), + } +} + +fn pixel_format_to_pixel(format: cv::PixelFormat) -> format::Pixel { + match format { + cv::PixelFormat::_420V => format::Pixel::NV12, + cv::PixelFormat::_32_RGBA => format::Pixel::RGBA, + _ => todo!(), + } +} diff --git a/crates/rendering/src/decoder/ffmpeg.rs b/crates/rendering/src/decoder/ffmpeg.rs new file mode 100644 index 00000000..215d919a --- /dev/null +++ b/crates/rendering/src/decoder/ffmpeg.rs @@ -0,0 +1,380 @@ +use std::{ + cell::LazyCell, + collections::BTreeMap, + path::PathBuf, + sync::{mpsc, Arc}, +}; + +use ffmpeg::{ + codec::{self, Capabilities}, + format, frame, rescale, software, Codec, Rescale, +}; +use ffmpeg_hw_device::{CodecContextExt, HwDevice}; +use ffmpeg_sys_next::{avcodec_find_decoder, AVHWDeviceType}; + +use super::{pts_to_frame, DecodedFrame, VideoDecoderMessage, FRAME_CACHE_SIZE}; + +#[derive(Clone)] +struct CachedFrame { + data: CachedFrameData, +} + +impl CachedFrame { + fn process(&mut self, decoder: &codec::decoder::Video) -> Arc> { + match &mut self.data { + CachedFrameData::Raw(frame) => { + let rgb_frame = if frame.format() != format::Pixel::RGBA { + // Reinitialize the scaler with the new input format + let mut scaler = software::converter( + (decoder.width(), decoder.height()), + frame.format(), + format::Pixel::RGBA, + ) + .unwrap(); + + let mut rgb_frame = frame::Video::empty(); + scaler.run(&frame, &mut rgb_frame).unwrap(); + rgb_frame + } else { + std::mem::replace(frame, frame::Video::empty()) + }; + + let width = rgb_frame.width() as usize; + let height = rgb_frame.height() as usize; + let stride = rgb_frame.stride(0); + let data = rgb_frame.data(0); + + let expected_size = width * height * 4; + + let mut frame_buffer = Vec::with_capacity(expected_size); + + // account for stride > width + for line_data in data.chunks_exact(stride) { + frame_buffer.extend_from_slice(&line_data[0..width * 4]); + } + + let data = Arc::new(frame_buffer); + + self.data = CachedFrameData::Processed(data.clone()); + + data + } + CachedFrameData::Processed(data) => data.clone(), + } + } +} + +#[derive(Clone)] +enum CachedFrameData { + Raw(frame::Video), + Processed(Arc>), +} + +struct FfmpegDecoder; + +impl FfmpegDecoder { + fn spawn(name: &'static str, path: PathBuf, fps: u32, rx: mpsc::Receiver) { + std::thread::spawn(move || { + let mut input = ffmpeg::format::input(&path).unwrap(); + + let input_stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .ok_or("Could not find a video stream") + .unwrap(); + + let decoder_codec = + find_decoder(&input, &input_stream, input_stream.parameters().id()).unwrap(); + + let mut context = codec::context::Context::new_with_codec(decoder_codec); + context.set_parameters(input_stream.parameters()).unwrap(); + + let input_stream_index = input_stream.index(); + let time_base = input_stream.time_base(); + let frame_rate = input_stream.rate(); + + // Create a decoder for the video stream + let mut decoder = context.decoder().video().unwrap(); + + { + use codec::threading::{Config, Type}; + + let capabilities = decoder_codec.capabilities(); + + if capabilities.intersects(Capabilities::FRAME_THREADS) { + decoder.set_threading(Config::kind(Type::Frame)); + } else if capabilities.intersects(Capabilities::SLICE_THREADS) { + decoder.set_threading(Config::kind(Type::Slice)); + } else { + decoder.set_threading(Config::count(1)); + } + } + + let hw_device: Option = { + #[cfg(target_os = "macos")] + { + decoder + .try_use_hw_device( + AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX, + Pixel::NV12, + ) + .ok() + } + #[cfg(target_os = "windows")] + { + decoder + .try_use_hw_device(AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA, Pixel::NV12) + .ok() + } + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + None + }; + + use ffmpeg::format::Pixel; + + let mut temp_frame = ffmpeg::frame::Video::empty(); + + // let mut packets = input.packets().peekable(); + + let width = decoder.width(); + let height = decoder.height(); + let black_frame = LazyCell::new(|| Arc::new(vec![0; (width * height * 4) as usize])); + + let mut cache = BTreeMap::::new(); + // active frame is a frame that triggered decode. + // frames that are within render_more_margin of this frame won't trigger decode. + let mut last_active_frame = None::; + + let mut last_decoded_frame = None::; + let mut last_sent_frame = None::<(u32, DecodedFrame)>; + + let mut peekable_requests = PeekableReceiver { rx, peeked: None }; + + let mut packets = input.packets().peekable(); + + while let Ok(r) = peekable_requests.recv() { + match r { + VideoDecoderMessage::GetFrame(requested_time, sender) => { + let requested_frame = (requested_time * fps as f32).floor() as u32; + // sender.send(black_frame.clone()).ok(); + // continue; + + let mut sender = if let Some(cached) = cache.get_mut(&requested_frame) { + let data = cached.process(&decoder); + + sender.send(data.clone()).ok(); + last_sent_frame = Some((requested_frame, data)); + continue; + } else { + Some(sender) + }; + + let cache_min = requested_frame.saturating_sub(FRAME_CACHE_SIZE as u32 / 2); + let cache_max = requested_frame + FRAME_CACHE_SIZE as u32 / 2; + + if requested_frame <= 0 + || last_sent_frame + .as_ref() + .map(|last| { + requested_frame < last.0 || + // seek forward for big jumps. this threshold is arbitrary but should be derived from i-frames in future + requested_frame - last.0 > FRAME_CACHE_SIZE as u32 + }) + .unwrap_or(true) + { + let timestamp_us = + ((requested_frame as f32 / frame_rate.numerator() as f32) + * 1_000_000.0) as i64; + let position = timestamp_us.rescale((1, 1_000_000), rescale::TIME_BASE); + + decoder.flush(); + input.seek(position, ..position).unwrap(); + last_decoded_frame = None; + + packets = input.packets().peekable(); + } + + last_active_frame = Some(requested_frame); + + loop { + if peekable_requests.peek().is_some() { + break; + } + let Some((stream, packet)) = packets.next() else { + // handles the case where the cache doesn't contain a frame so we fallback to the previously sent one + if let Some(last_sent_frame) = &last_sent_frame { + if last_sent_frame.0 < requested_frame { + sender.take().map(|s| s.send(last_sent_frame.1.clone())); + } + } + + sender.take().map(|s| s.send(black_frame.clone())); + break; + }; + + if stream.index() == input_stream_index { + let start_offset = stream.start_time(); + + decoder.send_packet(&packet).ok(); // decode failures are ok, we just fail to return a frame + + let mut exit = false; + + while decoder.receive_frame(&mut temp_frame).is_ok() { + let current_frame = pts_to_frame( + temp_frame.pts().unwrap() - start_offset, + time_base, + fps, + ); + + // Handles frame skips. requested_frame == last_decoded_frame should be handled by the frame cache. + if let Some((last_decoded_frame, sender)) = last_decoded_frame + .filter(|last_decoded_frame| { + requested_frame > *last_decoded_frame + && requested_frame < current_frame + }) + .and_then(|l| Some((l, sender.take()?))) + { + let data = cache + .get_mut(&last_decoded_frame) + .map(|f| f.process(&decoder)) + .unwrap_or_else(|| black_frame.clone()); + + last_sent_frame = Some((last_decoded_frame, data.clone())); + sender.send(data).ok(); + } + + last_decoded_frame = Some(current_frame); + + let exceeds_cache_bounds = current_frame > cache_max; + let too_small_for_cache_bounds = current_frame < cache_min; + + let hw_frame = + hw_device.as_ref().and_then(|d| d.get_hwframe(&temp_frame)); + + let frame = hw_frame.unwrap_or(std::mem::replace( + &mut temp_frame, + frame::Video::empty(), + )); + + if !too_small_for_cache_bounds { + let mut cache_frame = CachedFrame { + data: CachedFrameData::Raw(frame), + }; + + if current_frame == requested_frame { + if let Some(sender) = sender.take() { + let data = cache_frame.process(&decoder); + last_sent_frame = + Some((current_frame, data.clone())); + sender.send(data).ok(); + + break; + } + } + + if cache.len() >= FRAME_CACHE_SIZE { + if let Some(last_active_frame) = &last_active_frame { + let frame = if requested_frame > *last_active_frame + { + *cache.keys().next().unwrap() + } else if requested_frame < *last_active_frame { + *cache.keys().next_back().unwrap() + } else { + let min = *cache.keys().min().unwrap(); + let max = *cache.keys().max().unwrap(); + + if current_frame > max { + min + } else { + max + } + }; + + cache.remove(&frame); + } else { + cache.clear() + } + } + + cache.insert(current_frame, cache_frame); + } + + exit = exit || exceeds_cache_bounds; + } + + if exit { + break; + } + } + } + + if let Some((sender, last_sent_frame)) = + sender.take().zip(last_sent_frame.clone()) + { + sender.send(last_sent_frame.1).ok(); + } + } + } + } + }); + } +} + +pub fn find_decoder( + s: &format::context::Input, + st: &format::stream::Stream, + codec_id: codec::Id, +) -> Option { + unsafe { + use ffmpeg::media::Type; + let codec = match st.parameters().medium() { + Type::Video => Some((*s.as_ptr()).video_codec), + Type::Audio => Some((*s.as_ptr()).audio_codec), + Type::Subtitle => Some((*s.as_ptr()).subtitle_codec), + _ => None, + }; + + if let Some(codec) = codec { + if !codec.is_null() { + return Some(Codec::wrap(codec)); + } + } + + let found = avcodec_find_decoder(codec_id.into()); + + if found.is_null() { + return None; + } + Some(Codec::wrap(found)) + } +} + +struct PeekableReceiver { + rx: mpsc::Receiver, + peeked: Option, +} + +impl PeekableReceiver { + fn peek(&mut self) -> Option<&T> { + if self.peeked.is_some() { + self.peeked.as_ref() + } else { + match self.rx.try_recv() { + Ok(value) => { + self.peeked = Some(value); + self.peeked.as_ref() + } + Err(_) => None, + } + } + } + + fn recv(&mut self) -> Result { + if let Some(value) = self.peeked.take() { + Ok(value) + } else { + self.rx.recv() + } + } +} diff --git a/crates/rendering/src/decoder/mod.rs b/crates/rendering/src/decoder/mod.rs new file mode 100644 index 00000000..78630e1c --- /dev/null +++ b/crates/rendering/src/decoder/mod.rs @@ -0,0 +1,55 @@ +use ::ffmpeg::Rational; +use std::{ + path::PathBuf, + sync::{mpsc, Arc}, +}; + +#[cfg(target_os = "macos")] +mod avassetreader; +mod ffmpeg; + +pub type DecodedFrame = Arc>; + +pub enum VideoDecoderMessage { + GetFrame(f32, tokio::sync::oneshot::Sender), +} + +pub fn pts_to_frame(pts: i64, time_base: Rational, fps: u32) -> u32 { + (fps as f64 * ((pts as f64 * time_base.numerator() as f64) / (time_base.denominator() as f64))) + .round() as u32 +} + +pub const FRAME_CACHE_SIZE: usize = 100; + +#[derive(Clone)] +pub struct AsyncVideoDecoderHandle { + sender: mpsc::Sender, +} + +impl AsyncVideoDecoderHandle { + pub async fn get_frame(&self, time: f32) -> Option { + let (tx, rx) = tokio::sync::oneshot::channel(); + self.sender + .send(VideoDecoderMessage::GetFrame(time, tx)) + .unwrap(); + rx.await.ok() + } +} + +pub fn spawn_decoder(name: &'static str, path: PathBuf, fps: u32) -> AsyncVideoDecoderHandle { + let (tx, rx) = mpsc::channel(); + + let handle = AsyncVideoDecoderHandle { sender: tx }; + + #[cfg(target_os = "macos")] + { + avassetreader::AVAssetReaderDecoder::spawn(name, path, fps, rx); + } + + #[cfg(not(target_os = "macos"))] + { + FfmpegDecoder::spawn(name, path, fps, rx); + } + + handle +} diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 2bbab936..249a4540 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -9,7 +9,7 @@ use cap_project::{ SLOW_VELOCITY_THRESHOLD, XY, }; use core::f64; -use decoder::{AsyncVideoDecoder, AsyncVideoDecoderHandle, Decoder, GetFrameError}; +use decoder::{spawn_decoder, AsyncVideoDecoderHandle}; use futures::future::OptionFuture; use futures_intrusive::channel::shared::oneshot_channel; use serde::{Deserialize, Serialize}; @@ -20,7 +20,7 @@ use tokio::sync::mpsc; use wgpu::util::DeviceExt; use wgpu::{CommandEncoder, COPY_BYTES_PER_ROW_ALIGNMENT}; -use image::{GenericImageView, ImageDecoderRect}; +use image::GenericImageView; use std::path::Path; use std::time::Instant; @@ -100,7 +100,7 @@ pub struct SegmentVideoPaths<'a> { impl RecordingSegmentDecoders { pub fn new(meta: &RecordingMeta, segment: SegmentVideoPaths) -> Self { - let screen = Decoder::spawn( + let screen = spawn_decoder( "screen", meta.project_path.join(segment.display), match &meta.content { @@ -109,7 +109,7 @@ impl RecordingSegmentDecoders { }, ); let camera = segment.camera.map(|camera| { - Decoder::spawn( + spawn_decoder( "camera", meta.project_path.join(camera), match &meta.content { @@ -549,7 +549,6 @@ pub struct ProjectUniforms { pub cursor_size: f32, display: CompositeVideoFrameUniforms, camera: Option, - pub zoom: Zoom, pub project: ProjectConfiguration, pub is_upgraded: bool, } @@ -713,59 +712,43 @@ impl ProjectUniforms { &project.cursor.animation_style, ); - let zoom_keyframes = ZoomKeyframes::new(project); - let current_zoom = zoom_keyframes.interpolate(time as f64); - let prev_zoom = zoom_keyframes.interpolate((time - 1.0 / 30.0) as f64); - - let velocity = if current_zoom.amount != prev_zoom.amount { - let scale_change = (current_zoom.amount - prev_zoom.amount) as f32; - // Reduce the velocity scale from 0.05 to 0.02 - [ - (scale_change * output_size.0 as f32) * 0.02, // Reduced from 0.05 - (scale_change * output_size.1 as f32) * 0.02, - ] - } else { - [0.0, 0.0] - }; - - let motion_blur_amount = if current_zoom.amount != prev_zoom.amount { - project.motion_blur.unwrap_or(0.2) // Reduced from 0.5 to 0.2 - } else { - 0.0 - }; + // let zoom_keyframes = ZoomKeyframes::new(project); + // let current_zoom = zoom_keyframes.interpolate(time as f64); + // let prev_zoom = zoom_keyframes.interpolate((time - 1.0 / 30.0) as f64); + + let velocity = [0.0, 0.0]; + // if current_zoom.amount != prev_zoom.amount { + // let scale_change = (current_zoom.amount - prev_zoom.amount) as f32; + // // Reduce the velocity scale from 0.05 to 0.02 + // [ + // (scale_change * output_size.0 as f32) * 0.02, // Reduced from 0.05 + // (scale_change * output_size.1 as f32) * 0.02, + // ] + // } else { + // [0.0, 0.0] + // }; + + let motion_blur_amount = 0.0; + // if current_zoom.amount != prev_zoom.amount { + // project.motion_blur.unwrap_or(0.2) // Reduced from 0.5 to 0.2 + // } else { + // 0.0 + // }; let crop = Self::get_crop(options, project); - let interpolated_zoom = zoom_keyframes.interpolate(time as f64); - - let (zoom_amount, zoom_origin, lowered_zoom) = { - let origin = match interpolated_zoom.position { - ZoomPosition::Manual { x, y } => Coord::::new(XY { - x: x as f64, - y: y as f64, - }) - .to_raw_display_space(options) - .to_cropped_display_space(options, project), - ZoomPosition::Cursor => { - if let Some(cursor_position) = cursor_position { - cursor_position - .to_raw_display_space(options) - .to_cropped_display_space(options, project) - } else { - let center = XY::new( - options.screen_size.x as f64 / 2.0, - options.screen_size.y as f64 / 2.0, - ); - Coord::::new(center) - .to_cropped_display_space(options, project) - } - } - }; + let segment_cursor = SegmentsCursor::new( + time as f64, + project + .timeline + .as_ref() + .map(|t| t.zoom_segments.as_slice()) + .unwrap_or(&[]), + ); - (interpolated_zoom.amount, origin, interpolated_zoom.lowered) - }; + let zoom = InterpolatedZoom::new(segment_cursor); - let (display, zoom) = { + let display = { let output_size = XY::new(output_size.0 as f64, output_size.1 as f64); let size = [options.screen_size.x as f32, options.screen_size.y as f32]; @@ -782,27 +765,11 @@ impl ProjectUniforms { let end = Coord::new(output_size) - display_offset; - let screen_scale_origin = zoom_origin - .to_frame_space(options, project, resolution_base) - .clamp(display_offset.coord, end.coord); - - let zoom = Zoom { - amount: zoom_amount, - zoom_origin: screen_scale_origin, - // padding: screen_scale_origin, - }; - let target_size = end - display_offset; let (zoom_start, zoom_end) = ( - Coord::new(XY::new( - lowered_zoom.top_left.0 as f64 * target_size.x, - lowered_zoom.top_left.1 as f64 * target_size.y, - )), - Coord::new(XY::new( - (lowered_zoom.bottom_right.0 as f64 - 1.0) * target_size.x, - (lowered_zoom.bottom_right.1 as f64 - 1.0) * target_size.y, - )), + Coord::new(zoom.bounds.top_left * target_size.coord), + Coord::new((zoom.bounds.bottom_right - 1.0) * target_size.coord), ); let start = display_offset + zoom_start; @@ -811,28 +778,24 @@ impl ProjectUniforms { let target_size = end - start; let min_target_axis = target_size.x.min(target_size.y); - ( - CompositeVideoFrameUniforms { - output_size: [output_size.x as f32, output_size.y as f32], - frame_size: size, - crop_bounds: [ - crop_start.x as f32, - crop_start.y as f32, - crop_end.x as f32, - crop_end.y as f32, - ], - target_bounds: [start.x as f32, start.y as f32, end.x as f32, end.y as f32], - target_size: [target_size.x as f32, target_size.y as f32], - rounding_px: (project.background.rounding / 100.0 * 0.5 * min_target_axis) - as f32, - mirror_x: 0.0, - velocity_uv: velocity, - motion_blur_amount, - camera_motion_blur_amount: 0.0, - _padding: [0.0; 4], - }, - zoom, - ) + CompositeVideoFrameUniforms { + output_size: [output_size.x as f32, output_size.y as f32], + frame_size: size, + crop_bounds: [ + crop_start.x as f32, + crop_start.y as f32, + crop_end.x as f32, + crop_end.y as f32, + ], + target_bounds: [start.x as f32, start.y as f32, end.x as f32, end.y as f32], + target_size: [target_size.x as f32, target_size.y as f32], + rounding_px: (project.background.rounding / 100.0 * 0.5 * min_target_axis) as f32, + mirror_x: 0.0, + velocity_uv: velocity, + motion_blur_amount, + camera_motion_blur_amount: 0.0, + _padding: [0.0; 4], + } }; let camera = options @@ -847,8 +810,8 @@ impl ProjectUniforms { let base_size = project.camera.size / 100.0; let zoom_size = project.camera.zoom_size.unwrap_or(60.0) / 100.0; - let zoomed_size = (interpolated_zoom.t as f32) * zoom_size * base_size - + (1.0 - interpolated_zoom.t as f32) * base_size; + let zoomed_size = + (zoom.t as f32) * zoom_size * base_size + (1.0 - zoom.t as f32) * base_size; let size = [ min_axis * zoomed_size + CAMERA_PADDING, @@ -877,17 +840,18 @@ impl ProjectUniforms { ]; // Calculate camera motion blur based on zoom transition - let camera_motion_blur = { - let base_blur = project.motion_blur.unwrap_or(0.2); - let zoom_delta = (current_zoom.amount - prev_zoom.amount).abs() as f32; + let camera_motion_blur = 0.0; + // { + // let base_blur = project.motion_blur.unwrap_or(0.2); + // let zoom_delta = (current_zoom.amount - prev_zoom.amount).abs() as f32; - // Calculate a smooth transition factor - let transition_speed = 30.0f32; // Frames per second - let transition_factor = (zoom_delta * transition_speed).min(1.0); + // // Calculate a smooth transition factor + // let transition_speed = 30.0f32; // Frames per second + // let transition_factor = (zoom_delta * transition_speed).min(1.0); - // Reduce multiplier from 3.0 to 2.0 for weaker blur - (base_blur * 2.0 * transition_factor).min(1.0) - }; + // // Reduce multiplier from 3.0 to 2.0 for weaker blur + // (base_blur * 2.0 * transition_factor).min(1.0) + // }; CompositeVideoFrameUniforms { output_size, @@ -917,7 +881,6 @@ impl ProjectUniforms { cursor_size: project.cursor.size as f32, display, camera, - zoom, project: project.clone(), is_upgraded, } @@ -1401,7 +1364,8 @@ fn draw_cursor( let frame_position = cursor_position.to_frame_space(&constants.options, &uniforms.project, resolution_base); - let position = uniforms.zoom.apply_scale(frame_position); + // let position = uniforms.zoom.apply_scale(frame_position); + let position = frame_position; let relative_position = [position.x as f32, position.y as f32]; let cursor_uniforms = CursorUniforms { diff --git a/crates/rendering/src/zoom.rs b/crates/rendering/src/zoom.rs index 298b6afa..2b2ff1f3 100644 --- a/crates/rendering/src/zoom.rs +++ b/crates/rendering/src/zoom.rs @@ -1,275 +1,213 @@ -use cap_flags::FLAGS; -use cap_project::{ProjectConfiguration, ZoomSegment}; - -#[derive(Debug, PartialEq)] -pub struct ZoomKeyframe { - pub time: f64, - pub scale: f64, - pub position: ZoomPosition, - pub has_segment: bool, - pub lowered: LoweredKeyframe, -} -#[derive(Debug, PartialEq, Clone, Copy)] -pub enum ZoomPosition { - Cursor, - Manual { x: f32, y: f32 }, -} -#[derive(Debug, PartialEq)] -pub struct ZoomKeyframes(Vec); +use cap_project::{ZoomSegment, XY}; pub const ZOOM_DURATION: f64 = 1.0; -#[derive(Debug, PartialEq, Clone, Copy)] -pub struct InterpolatedZoom { - pub amount: f64, - pub t: f64, - pub position: ZoomPosition, - pub time_t: f64, - pub lowered: LoweredKeyframe, +#[derive(Debug, Clone, Copy)] +pub struct SegmentsCursor<'a> { + time: f64, + segment: Option<&'a ZoomSegment>, + prev_segment: Option<&'a ZoomSegment>, + segments: &'a [ZoomSegment], } -impl ZoomKeyframes { - pub fn new(config: &ProjectConfiguration) -> Self { - let Some(zoom_segments) = config.timeline().map(|t| &t.zoom_segments) else { - return Self(vec![]); - }; - - Self::from_zoom_segments(zoom_segments) - } - - fn from_zoom_segments(segments: &[ZoomSegment]) -> Self { - if segments.is_empty() { - return Self(vec![]); - } - - let mut keyframes = vec![]; - - for (i, segment) in segments.iter().enumerate() { - let position = match segment.mode { - cap_project::ZoomMode::Auto => ZoomPosition::Cursor, - cap_project::ZoomMode::Manual { x, y } => ZoomPosition::Manual { x, y }, - }; - - let prev = if i > 0 { segments.get(i - 1) } else { None }; - let next = segments.get(i + 1); - - let lowered_position = match segment.mode { - cap_project::ZoomMode::Auto => (0.0, 0.0), - cap_project::ZoomMode::Manual { x, y } => (x, y), - }; - - if let Some(prev) = prev { - if prev.end + ZOOM_DURATION < segment.start { - // keyframes.push(ZoomKeyframe { - // time: segment.start, - // scale: 1.0, - // position, - // }); - } - - keyframes.push(ZoomKeyframe { - time: segment.start + ZOOM_DURATION, - scale: segment.amount, - position, - has_segment: true, - lowered: LoweredKeyframe::new(lowered_position, segment.amount as f32), - }); - } else { - if segment.start != 0.0 { - keyframes.extend([ - ZoomKeyframe { - time: 0.0, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.0, y: 0.0 }, - has_segment: false, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0), - }, - ZoomKeyframe { - time: segment.start, - scale: 1.0, - position, - has_segment: true, - lowered: LoweredKeyframe::new(lowered_position, 1.0), - }, - ZoomKeyframe { - time: segment.start + ZOOM_DURATION, - scale: segment.amount, - position, - has_segment: true, - lowered: LoweredKeyframe::new(lowered_position, segment.amount as f32), - }, - ]); +impl<'a> SegmentsCursor<'a> { + pub fn new(time: f64, segments: &'a [ZoomSegment]) -> Self { + match segments + .iter() + .position(|s| time > s.start && time <= s.end) + { + Some(segment_index) => SegmentsCursor { + time, + segment: Some(&segments[segment_index]), + prev_segment: if segment_index > 0 { + Some(&segments[segment_index - 1]) } else { - keyframes.push(ZoomKeyframe { - time: segment.start, - scale: segment.amount, - position, - has_segment: true, - lowered: LoweredKeyframe::new(lowered_position, segment.amount as f32), - }); - } - } - - keyframes.push(ZoomKeyframe { - time: segment.end, - scale: segment.amount, - position, - has_segment: true, - lowered: LoweredKeyframe::new(lowered_position, segment.amount as f32), - }); - - if let Some(next) = next { - if segment.end + ZOOM_DURATION > next.start && next.start > segment.end { - let time = next.start - segment.end; - let t = time / ZOOM_DURATION; - - keyframes.push(ZoomKeyframe { - time: segment.end + time, - scale: 1.0 * t + (1.0 - t) * segment.amount, - position, - has_segment: false, - lowered: LoweredKeyframe::new( - lowered_position, - (1.0 * t + (1.0 - t) * segment.amount) as f32, - ), - }); + None + }, + segments, + }, + None => { + let prev = segments + .iter() + .enumerate() + .rev() + .find(|(_, s)| s.end <= time); + SegmentsCursor { + time, + segment: None, + prev_segment: prev.map(|(_, s)| s), + segments, } - } else { - keyframes.push(ZoomKeyframe { - time: segment.end + ZOOM_DURATION, - scale: 1.0, - position, - has_segment: false, - lowered: LoweredKeyframe::new(lowered_position, 1.0), - }); } } - - Self(keyframes) } +} - pub fn interpolate(&self, time: f64) -> InterpolatedZoom { - let default = InterpolatedZoom { - amount: 1.0, - position: ZoomPosition::Manual { x: 0.0, y: 0.0 }, - t: 0.0, - time_t: 0.0, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0), - }; - - if !FLAGS.zoom { - return default; - } - - let prev_index = self - .0 - .iter() - .rev() - .position(|k| time >= k.time) - .map(|p| self.0.len() - 1 - p); +#[derive(Debug, PartialEq, Clone, Copy)] +pub struct SegmentBounds { + pub top_left: XY, + pub bottom_right: XY, +} - let Some(prev_index) = prev_index else { - return default; +impl SegmentBounds { + fn from_segment(segment: &ZoomSegment) -> Self { + let position = match segment.mode { + cap_project::ZoomMode::Auto => (0.0, 0.0), + cap_project::ZoomMode::Manual { x, y } => (x, y), }; - let next_index = prev_index + 1; - - let Some((prev, next)) = self.0.get(prev_index).zip(self.0.get(next_index)) else { - return default; - }; + let scaled_center = [ + position.0 as f64 * segment.amount, + position.1 as f64 * segment.amount, + ]; + let center_diff = [ + scaled_center[0] - position.0 as f64, + scaled_center[1] - position.1 as f64, + ]; - let keyframe_length = next.time - prev.time; - let delta_time = time - prev.time; + SegmentBounds::new( + XY::new(0.0 - center_diff[0], 0.0 - center_diff[1]), + XY::new( + segment.amount - center_diff[0], + segment.amount - center_diff[1], + ), + ) + } - let ease = if next.scale >= prev.scale { - bezier_easing::bezier_easing(0.1, 0.0, 0.3, 1.0).unwrap() - } else { - bezier_easing::bezier_easing(0.5, 0.0, 0.5, 1.0).unwrap() - }; + pub fn new(top_left: XY, bottom_right: XY) -> Self { + Self { + top_left, + bottom_right, + } + } - let time_t_raw = delta_time / keyframe_length; + pub fn default() -> Self { + SegmentBounds::new(XY::new(0.0, 0.0), XY::new(1.0, 1.0)) + } +} - let keyframe_diff = next.scale - prev.scale; +#[derive(Debug, Clone, Copy)] +pub struct InterpolatedZoom { + // the ratio of current zoom to the maximum amount for the current segment + pub t: f64, + pub bounds: SegmentBounds, +} - // let time_t = ease(time_t_raw as f32) as f64; - let time_t = time_t_raw; +impl InterpolatedZoom { + pub fn new(cursor: SegmentsCursor) -> Self { + let ease_in = bezier_easing::bezier_easing(0.1, 0.0, 0.3, 1.0).unwrap(); + let ease_out = bezier_easing::bezier_easing(0.5, 0.0, 0.5, 1.0).unwrap(); - let amount = prev.scale + (keyframe_diff) * time_t; + Self::new_with_easing(cursor, ease_in, ease_out) + } - // the process we use to get to this is way too convoluted lol - let t = if prev.scale > 1.0 && next.scale > 1.0 { - if !next.has_segment { - (amount - 1.0) / (prev.scale - 1.0) - } else if !prev.has_segment { - (amount - 1.0) / (next.scale - 1.0) - } else { - 1.0 + pub(self) fn new_with_easing( + cursor: SegmentsCursor, + ease_in: impl Fn(f32) -> f32, + ease_out: impl Fn(f32) -> f32, + ) -> InterpolatedZoom { + let default = SegmentBounds::default(); + match (cursor.prev_segment, cursor.segment) { + (Some(prev_segment), None) => { + let zoom_t = + ease_out(t_clamp((cursor.time - prev_segment.end) / ZOOM_DURATION) as f32) + as f64; + + Self { + t: 1.0 - zoom_t, + bounds: { + let prev_segment_bounds = SegmentBounds::from_segment(prev_segment); + + SegmentBounds::new( + prev_segment_bounds.top_left * (1.0 - zoom_t) + + default.top_left * zoom_t, + prev_segment_bounds.bottom_right * (1.0 - zoom_t) + + default.bottom_right * zoom_t, + ) + }, + } } - } else if next.scale > 1.0 { - (amount - 1.0) / (next.scale - 1.0) - } else if prev.scale > 1.0 { - (amount - 1.0) / (prev.scale - 1.0) - } else { - 0.0 - }; - - let position = match (&prev.position, &next.position) { - (ZoomPosition::Manual { x: x1, y: y1 }, ZoomPosition::Manual { x: x2, y: y2 }) => { - ZoomPosition::Manual { - x: x1 + (x2 - x1) * time_t_raw as f32, - y: y1 + (y2 - y1) * time_t_raw as f32, + (None, Some(segment)) => { + let t = + ease_in(t_clamp((cursor.time - segment.start) / ZOOM_DURATION) as f32) as f64; + + Self { + t, + bounds: { + let segment_bounds = SegmentBounds::from_segment(segment); + + SegmentBounds::new( + default.top_left * (1.0 - t) + segment_bounds.top_left * t, + default.bottom_right * (1.0 - t) + segment_bounds.bottom_right * t, + ) + }, } } - _ => ZoomPosition::Manual { x: 0.0, y: 0.0 }, - }; - - let eased_time_t = ease(time_t as f32); - - InterpolatedZoom { - time_t, - amount: prev.scale + (next.scale - prev.scale) * time_t, - position, - t: ease(t as f32) as f64, - lowered: LoweredKeyframe { - top_left: { - let prev = prev.lowered.top_left; - let next = next.lowered.top_left; - - ( - prev.0 + (next.0 - prev.0) * eased_time_t, - prev.1 + (next.1 - prev.1) * eased_time_t, - ) - }, - bottom_right: { - let prev = prev.lowered.bottom_right; - let next = next.lowered.bottom_right; - - ( - prev.0 + (next.0 - prev.0) * eased_time_t, - prev.1 + (next.1 - prev.1) * eased_time_t, - ) - }, + (Some(prev_segment), Some(segment)) => { + let prev_segment_bounds = SegmentBounds::from_segment(prev_segment); + let segment_bounds = SegmentBounds::from_segment(segment); + + let zoom_t = + ease_in(t_clamp((cursor.time - segment.start) / ZOOM_DURATION) as f32) as f64; + + // no gap + if segment.start == prev_segment.end { + Self { + t: 1.0, + bounds: SegmentBounds::new( + prev_segment_bounds.top_left * (1.0 - zoom_t) + + segment_bounds.top_left * zoom_t, + prev_segment_bounds.bottom_right * (1.0 - zoom_t) + + segment_bounds.bottom_right * zoom_t, + ), + } + } + // small gap + else if segment.start - prev_segment.end < ZOOM_DURATION { + // handling this is a bit funny, since we're not zooming in from 0 but rather + // from the previous value that the zoom out got interrupted at by the current segment + + let min = InterpolatedZoom::new_with_easing( + SegmentsCursor::new(segment.start, cursor.segments), + ease_in, + ease_out, + ); + + Self { + t: (min.t * (1.0 - zoom_t)) + zoom_t, + bounds: { + let max = segment_bounds; + + SegmentBounds::new( + min.bounds.top_left * (1.0 - zoom_t) + max.top_left * zoom_t, + min.bounds.bottom_right * (1.0 - zoom_t) + + max.bottom_right * zoom_t, + ) + }, + } + } + // entirely separate + else { + Self { + t: zoom_t, + bounds: SegmentBounds::new( + default.top_left * (1.0 - zoom_t) + segment_bounds.top_left * zoom_t, + default.bottom_right * (1.0 - zoom_t) + + segment_bounds.bottom_right * zoom_t, + ), + } + } + } + _ => Self { + t: 0.0, + bounds: default, }, } } } -#[derive(Debug, PartialEq, Clone, Copy)] -pub struct LoweredKeyframe { - pub top_left: (f32, f32), - pub bottom_right: (f32, f32), -} - -impl LoweredKeyframe { - fn new(center: (f32, f32), amount: f32) -> Self { - let scaled_center = (center.0 * amount, center.1 * amount); - let center_diff = (scaled_center.0 - center.0, scaled_center.1 - center.1); - - Self { - top_left: (0.0 - center_diff.0, 0.0 - center_diff.1), - bottom_right: (amount - center_diff.0, amount - center_diff.1), - } - } +fn t_clamp(v: f64) -> f64 { + v.clamp(0.0, 1.0) } #[cfg(test)] @@ -278,437 +216,275 @@ mod test { use super::*; - #[test] - fn single_keyframe() { - let segments = [ZoomSegment { - start: 0.5, - end: 1.5, - amount: 1.5, - mode: cap_project::ZoomMode::Manual { x: 0.2, y: 0.2 }, - }]; + // Custom macro for floating-point near equality + macro_rules! assert_f64_near { + ($left:expr, $right:expr, $label:literal) => { + let left = $left; + let right = $right; + assert!( + (left - right).abs() < 1e-6, + "{}: `(left ~ right)` \n left: `{:?}`, \n right: `{:?}`", + $label, + left, + right + ) + }; + ($left:expr, $right:expr) => { + assert_f64_near!($left, $right, "assertion failed"); + }; + } - let keyframes = ZoomKeyframes::from_zoom_segments(&segments); - - pretty_assertions::assert_eq!( - keyframes, - ZoomKeyframes(vec![ - ZoomKeyframe { - time: 0.0, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.0, y: 0.0 }, - has_segment: false, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0) - }, - ZoomKeyframe { - time: 0.5, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.0) - }, - ZoomKeyframe { - time: 0.5 + ZOOM_DURATION, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.5) - }, - ZoomKeyframe { - time: 1.5, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.5) - }, - ZoomKeyframe { - time: 1.5 + ZOOM_DURATION, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: false, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.0) - } - ]) - ); + fn c(time: f64, segments: &[ZoomSegment]) -> SegmentsCursor { + SegmentsCursor::new(time, segments) + } + + fn test_interp((time, segments): (f64, &[ZoomSegment]), expected: InterpolatedZoom) { + let actual = InterpolatedZoom::new_with_easing(c(time, segments), |t| t, |t| t); + + assert_f64_near!(actual.t, expected.t, "t"); + + let a = &actual.bounds; + let e = &expected.bounds; + + assert_f64_near!(a.top_left.x, e.top_left.x, "bounds.top_left.x"); + assert_f64_near!(a.top_left.y, e.top_left.y, "bounds.top_left.y"); + assert_f64_near!(a.bottom_right.x, e.bottom_right.x, "bounds.bottom_right.x"); + assert_f64_near!(a.bottom_right.y, e.bottom_right.y, "bounds.bottom_right.y"); } #[test] - fn adjancent_different_position() { - let segments = [ - ZoomSegment { - start: 0.5, - end: 1.5, - amount: 1.5, - mode: cap_project::ZoomMode::Manual { x: 0.2, y: 0.2 }, + fn one_segment() { + let segments = vec![ZoomSegment { + start: 2.0, + end: 4.0, + amount: 2.0, + mode: ZoomMode::Manual { x: 0.5, y: 0.5 }, + }]; + + test_interp( + (0.0, &segments), + InterpolatedZoom { + t: 0.0, + bounds: SegmentBounds::default(), }, - ZoomSegment { - start: 1.5, - end: 2.5, - amount: 1.5, - mode: cap_project::ZoomMode::Manual { x: 0.8, y: 0.8 }, + ); + test_interp( + (2.0, &segments), + InterpolatedZoom { + t: 0.0, + bounds: SegmentBounds::default(), + }, + ); + test_interp( + (2.0 + ZOOM_DURATION * 0.1, &segments), + InterpolatedZoom { + t: 0.1, + bounds: SegmentBounds::new(XY::new(-0.05, -0.05), XY::new(1.05, 1.05)), + }, + ); + test_interp( + (2.0 + ZOOM_DURATION * 0.9, &segments), + InterpolatedZoom { + t: 0.9, + bounds: SegmentBounds::new(XY::new(-0.45, -0.45), XY::new(1.45, 1.45)), + }, + ); + test_interp( + (2.0 + ZOOM_DURATION, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-0.5, -0.5), XY::new(1.5, 1.5)), + }, + ); + test_interp( + (4.0, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-0.5, -0.5), XY::new(1.5, 1.5)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * 0.2, &segments), + InterpolatedZoom { + t: 0.8, + bounds: SegmentBounds::new(XY::new(-0.4, -0.4), XY::new(1.4, 1.4)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * 0.8, &segments), + InterpolatedZoom { + t: 0.2, + bounds: SegmentBounds::new(XY::new(-0.1, -0.1), XY::new(1.1, 1.1)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION, &segments), + InterpolatedZoom { + t: 0.0, + bounds: SegmentBounds::new(XY::new(0.0, 0.0), XY::new(1.0, 1.0)), }, - ]; - - let keyframes = ZoomKeyframes::from_zoom_segments(&segments); - - pretty_assertions::assert_eq!( - keyframes, - ZoomKeyframes(vec![ - ZoomKeyframe { - time: 0.0, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.0, y: 0.0 }, - has_segment: false, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0) - }, - ZoomKeyframe { - time: 0.5, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.0) - }, - ZoomKeyframe { - time: 0.5 + ZOOM_DURATION, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.5) - }, - ZoomKeyframe { - time: 1.5, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.5) - }, - ZoomKeyframe { - time: 1.5 + ZOOM_DURATION, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.8, y: 0.8 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.8, 0.8), 1.5) - }, - ZoomKeyframe { - time: 2.5, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.8, y: 0.8 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.8, 0.8), 1.5) - }, - ZoomKeyframe { - time: 2.5 + ZOOM_DURATION, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.8, y: 0.8 }, - has_segment: false, - lowered: LoweredKeyframe::new((0.8, 0.8), 1.0) - } - ]) ); } #[test] - fn adjacent_different_amount() { - let segments = [ + fn two_segments_no_gap() { + let segments = vec![ ZoomSegment { - start: 0.5, - end: 1.5, - amount: 1.5, - mode: cap_project::ZoomMode::Manual { x: 0.2, y: 0.2 }, + start: 2.0, + end: 4.0, + amount: 2.0, + mode: ZoomMode::Manual { x: 0.0, y: 0.0 }, }, ZoomSegment { - start: 1.5, - end: 2.5, - amount: 2.0, - mode: cap_project::ZoomMode::Manual { x: 0.2, y: 0.2 }, + start: 4.0, + end: 6.0, + amount: 4.0, + mode: ZoomMode::Manual { x: 0.5, y: 0.5 }, }, ]; - let keyframes = ZoomKeyframes::from_zoom_segments(&segments); - - pretty_assertions::assert_eq!( - keyframes, - ZoomKeyframes(vec![ - ZoomKeyframe { - time: 0.0, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.0, y: 0.0 }, - has_segment: false, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0) - }, - ZoomKeyframe { - time: 0.5, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.0) - }, - ZoomKeyframe { - time: 0.5 + ZOOM_DURATION, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.5) - }, - ZoomKeyframe { - time: 1.5, - scale: 1.5, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.5) - }, - ZoomKeyframe { - time: 1.5 + ZOOM_DURATION, - scale: 2.0, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 2.0) - }, - ZoomKeyframe { - time: 2.5, - scale: 2.0, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: true, - lowered: LoweredKeyframe::new((0.2, 0.2), 2.0) - }, - ZoomKeyframe { - time: 2.5 + ZOOM_DURATION, - scale: 1.0, - position: ZoomPosition::Manual { x: 0.2, y: 0.2 }, - has_segment: false, - lowered: LoweredKeyframe::new((0.2, 0.2), 1.0) - } - ]) + test_interp( + (4.0, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(0.0, 0.0), XY::new(2.0, 2.0)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * 0.2, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-0.3, -0.3), XY::new(2.1, 2.1)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * 0.8, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-1.2, -1.2), XY::new(2.4, 2.4)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-1.5, -1.5), XY::new(2.5, 2.5)), + }, ); } #[test] - fn gap() { - let segments = [ + fn two_segments_small_gap() { + let segments = vec![ ZoomSegment { - start: 0.5, - end: 1.5, - amount: 1.5, - mode: cap_project::ZoomMode::Manual { x: 0.0, y: 0.0 }, + start: 2.0, + end: 4.0, + amount: 2.0, + mode: ZoomMode::Manual { x: 0.5, y: 0.5 }, }, ZoomSegment { - start: 1.8, - end: 2.5, - amount: 1.5, - mode: cap_project::ZoomMode::Manual { x: 0.0, y: 0.0 }, + start: 4.0 + ZOOM_DURATION * 0.75, + end: 6.0, + amount: 4.0, + mode: ZoomMode::Manual { x: 0.5, y: 0.5 }, }, ]; - let keyframes = ZoomKeyframes::from_zoom_segments(&segments); - - let position = ZoomPosition::Manual { x: 0.0, y: 0.0 }; - let base = ZoomKeyframe { - time: 0.0, - scale: 1.0, - position, - has_segment: true, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0), - }; - - pretty_assertions::assert_eq!( - keyframes, - ZoomKeyframes(vec![ - ZoomKeyframe { - has_segment: false, - ..base - }, - ZoomKeyframe { time: 0.5, ..base }, - ZoomKeyframe { - time: 0.5 + ZOOM_DURATION, - scale: 1.5, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.5), - ..base - }, - ZoomKeyframe { - time: 1.5, - scale: 1.5, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.5), - ..base - }, - ZoomKeyframe { - time: 1.8, - scale: 1.5 - (0.3 / ZOOM_DURATION) * 0.5, - lowered: LoweredKeyframe::new( - (0.0, 0.0), - 1.5 - (0.3 / ZOOM_DURATION as f32) * 0.5 - ), - has_segment: false, - ..base - }, - ZoomKeyframe { - time: 1.8 + ZOOM_DURATION, - scale: 1.5, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.5), - ..base - }, - ZoomKeyframe { - time: 2.5, - scale: 1.5, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.5), - ..base - }, - ZoomKeyframe { - time: 2.5 + ZOOM_DURATION, - scale: 1.0, - has_segment: false, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0), - ..base - } - ]) + test_interp( + (4.0, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-0.5, -0.5), XY::new(1.5, 1.5)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * 0.5, &segments), + InterpolatedZoom { + t: 0.5, + bounds: SegmentBounds::new(XY::new(-0.25, -0.25), XY::new(1.25, 1.25)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * 0.75, &segments), + InterpolatedZoom { + t: 0.25, + bounds: SegmentBounds::new(XY::new(-0.125, -0.125), XY::new(1.125, 1.125)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * (0.75 + 0.5), &segments), + InterpolatedZoom { + t: 0.625, + bounds: SegmentBounds::new(XY::new(-0.8125, -0.8125), XY::new(1.8125, 1.8125)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * (0.75 + 1.0), &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-1.5, -1.5), XY::new(2.5, 2.5)), + }, ); } #[test] - fn project_config() { - let segments = [ + fn two_segments_large_gap() { + let segments = vec![ ZoomSegment { - start: 0.3966305848375451, - end: 1.396630584837545, - amount: 1.176, - mode: cap_project::ZoomMode::Manual { x: 0.0, y: 0.0 }, + start: 2.0, + end: 4.0, + amount: 2.0, + mode: ZoomMode::Manual { x: 0.5, y: 0.5 }, }, ZoomSegment { - start: 1.396630584837545, - end: 3.21881273465704, - amount: 1.204, - mode: cap_project::ZoomMode::Manual { x: 0.0, y: 0.0 }, + start: 7.0, + end: 9.0, + amount: 4.0, + mode: ZoomMode::Manual { x: 0.0, y: 0.0 }, }, ]; - let keyframes = ZoomKeyframes::from_zoom_segments(&segments); - - let position = ZoomPosition::Manual { x: 0.0, y: 0.0 }; - let base = ZoomKeyframe { - time: 0.0, - scale: 1.0, - position, - has_segment: true, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.0), - }; - - pretty_assertions::assert_eq!( - keyframes, - ZoomKeyframes(vec![ - ZoomKeyframe { - has_segment: false, - ..base - }, - ZoomKeyframe { - time: 0.3966305848375451, - ..base - }, - ZoomKeyframe { - time: 0.3966305848375451 + ZOOM_DURATION, - scale: 1.176, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.176), - ..base - }, - ZoomKeyframe { - time: 1.396630584837545, - scale: 1.176, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.176), - ..base - }, - ZoomKeyframe { - time: 1.396630584837545 + ZOOM_DURATION, - scale: 1.204, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.204), - ..base - }, - ZoomKeyframe { - time: 3.21881273465704, - scale: 1.204, - lowered: LoweredKeyframe::new((0.0, 0.0), 1.204), - ..base - }, - ZoomKeyframe { - time: 3.21881273465704 + ZOOM_DURATION, - has_segment: false, - ..base - }, - ]) + test_interp( + (4.0, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(-0.5, -0.5), XY::new(1.5, 1.5)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION * 0.5, &segments), + InterpolatedZoom { + t: 0.5, + bounds: SegmentBounds::new(XY::new(-0.25, -0.25), XY::new(1.25, 1.25)), + }, + ); + test_interp( + (4.0 + ZOOM_DURATION, &segments), + InterpolatedZoom { + t: 0.0, + bounds: SegmentBounds::new(XY::new(0.0, 0.0), XY::new(1.0, 1.0)), + }, + ); + test_interp( + (7.0, &segments), + InterpolatedZoom { + t: 0.0, + bounds: SegmentBounds::new(XY::new(0.0, 0.0), XY::new(1.0, 1.0)), + }, + ); + test_interp( + (7.0 + ZOOM_DURATION * 0.5, &segments), + InterpolatedZoom { + t: 0.5, + bounds: SegmentBounds::new(XY::new(0.0, 0.0), XY::new(2.5, 2.5)), + }, + ); + test_interp( + (7.0 + ZOOM_DURATION * 1.0, &segments), + InterpolatedZoom { + t: 1.0, + bounds: SegmentBounds::new(XY::new(0.0, 0.0), XY::new(4.0, 4.0)), + }, ); - } - - mod interpolate { - use super::*; - - #[test] - fn amount() { - let keyframes = ZoomKeyframes::from_zoom_segments(&[ - ZoomSegment { - start: 0.0, - end: 1.0, - amount: 1.2, - mode: ZoomMode::Manual { x: 0.0, y: 0.0 }, - }, - ZoomSegment { - start: 1.0, - end: 2.0, - amount: 1.5, - mode: ZoomMode::Manual { x: 0.0, y: 0.0 }, - }, - ]); - - assert_eq!(keyframes.interpolate(0.0).amount, 1.2); - assert_eq!(keyframes.interpolate(1.0).amount, 1.2); - assert_eq!(keyframes.interpolate(2.0).amount, 1.5); - } - - #[test] - fn t() { - let keyframes = ZoomKeyframes::from_zoom_segments(&[ - ZoomSegment { - start: 0.0, - end: 1.0, - amount: 1.2, - mode: ZoomMode::Manual { x: 0.0, y: 0.0 }, - }, - ZoomSegment { - start: 1.0, - end: 2.0, - amount: 1.5, - mode: ZoomMode::Manual { x: 0.0, y: 0.0 }, - }, - ]); - - assert_eq!(keyframes.interpolate(0.0).t, 1.0); - assert_eq!(keyframes.interpolate(1.0).t, 1.0); - assert_eq!(keyframes.interpolate(2.0).t, 1.0); - assert_eq!(keyframes.interpolate(2.0 + ZOOM_DURATION).t, 0.0); - } - } - - mod new_keyframe_lowering { - use super::*; - - #[test] - fn basic() { - let center = (0.0, 0.0); - let amount = 2.0; - - assert_eq!( - LoweredKeyframe::new(center, amount), - LoweredKeyframe { - top_left: (0.0, 0.0), - bottom_right: (2.0, 2.0) - } - ); - - let center = (1.0, 1.0); - let amount = 2.0; - - assert_eq!( - LoweredKeyframe::new(center, amount), - LoweredKeyframe { - top_left: (-1.0, -1.0), - bottom_right: (1.0, 1.0) - } - ); - } } } diff --git a/package.json b/package.json index 33c09c85..721ac41d 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,8 @@ "eslint": "^7.32.0", "extract-zip": "^2.0.1", "prettier": "^2.5.1", - "turbo": "^1.10.16" + "turbo": "^1.10.16", + "typescript": "^5.7.2" }, "packageManager": "pnpm@9.8.0", "name": "cap", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 500f47ba..2df25332 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -31,6 +31,9 @@ importers: turbo: specifier: ^1.10.16 version: 1.13.4 + typescript: + specifier: ^5.7.2 + version: 5.7.2 apps/desktop: dependencies: @@ -421,7 +424,7 @@ importers: version: 8.57.1 eslint-config-airbnb-typescript: specifier: ^18.0.0 - version: 18.0.0(@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint@8.57.1)(typescript@5.7.2))(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0)(eslint@8.57.1) + version: 18.0.0(@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint@8.57.1)(typescript@5.7.2))(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1))(eslint@8.57.1) eslint-import-resolver-typescript: specifier: ^3.6.1 version: 3.6.3(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0)(eslint@8.57.1) @@ -4300,10 +4303,10 @@ packages: react-dom: optional: true - '@storybook/builder-vite@8.6.0-alpha.0': - resolution: {integrity: sha512-31OJfaS6Aj3vFO/0pJE6iz3sDkQU6TB0m6/WfZ+i80Re4OG1pu5GCvP8bqOFuEAQpyc2sFKs6B+wOBvZuuSskg==} + '@storybook/builder-vite@8.6.0-alpha.1': + resolution: {integrity: sha512-14mIpBx5oxjliGokiejGxUGReG0UNQwrkeu4FiObX8QQWl94/eCO9ExxdyYPNjMx7OfOfhqwIL3SQnoChp+2yA==} peerDependencies: - storybook: ^8.6.0-alpha.0 + storybook: ^8.6.0-alpha.1 vite: ^4.0.0 || ^5.0.0 || ^6.0.0 '@storybook/core@8.3.3': @@ -4314,10 +4317,10 @@ packages: peerDependencies: storybook: ^8.3.3 - '@storybook/csf-plugin@8.6.0-alpha.0': - resolution: {integrity: sha512-FkI3YW7ObEAFBQokJ04uPIQnBbnsfUKf08jriZ/nB03dcUNVDkl1GjPHONXJK8IwUKfKsiXRVqmrqJPSkK6xvQ==} + '@storybook/csf-plugin@8.6.0-alpha.1': + resolution: {integrity: sha512-Us0p0WaM4QUigPTQ9pRjfUM8PtuflPriuN51FFlTiJDmLF/OEmr/AvWnYhET6RZCYD4p912TVJN/t2xig+nMdA==} peerDependencies: - storybook: ^8.6.0-alpha.0 + storybook: ^8.6.0-alpha.1 '@storybook/csf@0.1.11': resolution: {integrity: sha512-dHYFQH3mA+EtnCkHXzicbLgsvzYjcDJ1JWsogbItZogkPHgSJM/Wr71uMkcvw8v9mmCyP4NpXJuu6bPoVsOnzg==} @@ -15052,9 +15055,9 @@ snapshots: react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - '@storybook/builder-vite@8.6.0-alpha.0(storybook@8.3.3)(vite@5.4.8(@types/node@20.16.9)(terser@5.34.0))': + '@storybook/builder-vite@8.6.0-alpha.1(storybook@8.3.3)(vite@5.4.8(@types/node@20.16.9)(terser@5.34.0))': dependencies: - '@storybook/csf-plugin': 8.6.0-alpha.0(storybook@8.3.3) + '@storybook/csf-plugin': 8.6.0-alpha.1(storybook@8.3.3) browser-assert: 1.2.1 storybook: 8.3.3 ts-dedent: 2.2.0 @@ -15085,7 +15088,7 @@ snapshots: storybook: 8.3.3 unplugin: 1.16.0 - '@storybook/csf-plugin@8.6.0-alpha.0(storybook@8.3.3)': + '@storybook/csf-plugin@8.6.0-alpha.1(storybook@8.3.3)': dependencies: storybook: 8.3.3 unplugin: 1.16.0 @@ -17646,7 +17649,7 @@ snapshots: optionalDependencies: source-map: 0.6.1 - eslint-config-airbnb-base@15.0.0(eslint-plugin-import@2.30.0)(eslint@8.57.1): + eslint-config-airbnb-base@15.0.0(eslint-plugin-import@2.30.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1))(eslint@8.57.1): dependencies: confusing-browser-globals: 1.0.11 eslint: 8.57.1 @@ -17655,12 +17658,12 @@ snapshots: object.entries: 1.1.8 semver: 6.3.1 - eslint-config-airbnb-typescript@18.0.0(@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint@8.57.1)(typescript@5.7.2))(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0)(eslint@8.57.1): + eslint-config-airbnb-typescript@18.0.0(@typescript-eslint/eslint-plugin@7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint@8.57.1)(typescript@5.7.2))(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1))(eslint@8.57.1): dependencies: '@typescript-eslint/eslint-plugin': 7.18.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint@8.57.1)(typescript@5.7.2) '@typescript-eslint/parser': 7.18.0(eslint@8.57.1)(typescript@5.7.2) eslint: 8.57.1 - eslint-config-airbnb-base: 15.0.0(eslint-plugin-import@2.30.0)(eslint@8.57.1) + eslint-config-airbnb-base: 15.0.0(eslint-plugin-import@2.30.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1))(eslint@8.57.1) transitivePeerDependencies: - eslint-plugin-import @@ -17744,7 +17747,7 @@ snapshots: debug: 4.3.7(supports-color@5.5.0) enhanced-resolve: 5.17.1 eslint: 8.57.1 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0)(eslint@8.57.1))(eslint@8.57.1) fast-glob: 3.3.2 get-tsconfig: 4.8.1 is-bun-module: 1.2.1 @@ -17763,7 +17766,7 @@ snapshots: debug: 4.3.7(supports-color@5.5.0) enhanced-resolve: 5.17.1 eslint: 8.57.1 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0)(eslint@8.57.1))(eslint@8.57.1) fast-glob: 3.3.2 get-tsconfig: 4.8.1 is-bun-module: 1.2.1 @@ -17787,7 +17790,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): + eslint-module-utils@2.12.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0)(eslint@8.57.1))(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: @@ -17798,7 +17801,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1): + eslint-module-utils@2.12.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0)(eslint@8.57.1))(eslint@8.57.1): dependencies: debug: 3.2.7 optionalDependencies: @@ -17820,7 +17823,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@5.62.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.30.0)(eslint@8.57.1))(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.15.1 is-glob: 4.0.3 @@ -17848,7 +17851,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.1 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3)(eslint@8.57.1) + eslint-module-utils: 2.12.0(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.3(@typescript-eslint/parser@7.18.0(eslint@8.57.1)(typescript@5.7.2))(eslint-plugin-import@2.30.0)(eslint@8.57.1))(eslint@8.57.1) hasown: 2.0.2 is-core-module: 2.15.1 is-glob: 4.0.3 @@ -22627,7 +22630,7 @@ snapshots: storybook-solidjs-vite@1.0.0-beta.2(storybook@8.3.3)(vite@5.4.8(@types/node@20.16.9)(terser@5.34.0)): dependencies: - '@storybook/builder-vite': 8.6.0-alpha.0(storybook@8.3.3)(vite@5.4.8(@types/node@20.16.9)(terser@5.34.0)) + '@storybook/builder-vite': 8.6.0-alpha.1(storybook@8.3.3)(vite@5.4.8(@types/node@20.16.9)(terser@5.34.0)) transitivePeerDependencies: - storybook - vite