diff --git a/Cargo.lock b/Cargo.lock index bce11d2cb0b3..c3767eabd3c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4946,7 +4946,6 @@ dependencies = [ "re_smart_channel", "re_tracing", "re_types", - "re_video", "thiserror", "walkdir", ] @@ -5798,8 +5797,10 @@ dependencies = [ name = "re_video" version = "0.19.0-alpha.1+dev" dependencies = [ + "itertools 0.13.0", "mp4", "ordered-float", + "thiserror", ] [[package]] @@ -6155,6 +6156,7 @@ dependencies = [ "re_arrow2", "re_log", "re_sdk", + "re_video", ] [[package]] @@ -6179,6 +6181,7 @@ dependencies = [ "re_log_types", "re_memory", "re_sdk", + "re_video", "re_web_viewer_server", "re_ws_comms", "uuid", diff --git a/crates/store/README.md b/crates/store/README.md index b4841b792024..2c2c71c97606 100644 --- a/crates/store/README.md +++ b/crates/store/README.md @@ -1 +1 @@ -Creates related to storing, indexing, trasmitting, and handling data. +Creates related to storing, indexing, transmitting, and handling data. diff --git a/crates/store/re_data_loader/Cargo.toml b/crates/store/re_data_loader/Cargo.toml index 804ff9889f46..b5f4afb54d96 100644 --- a/crates/store/re_data_loader/Cargo.toml +++ b/crates/store/re_data_loader/Cargo.toml @@ -31,8 +31,7 @@ re_log_types.workspace = true re_log.workspace = true re_smart_channel.workspace = true re_tracing.workspace = true -re_types = { workspace = true, features = ["image"] } -re_video.workspace = true +re_types = { workspace = true, features = ["image", "video"] } ahash.workspace = true anyhow.workspace = true diff --git a/crates/store/re_data_loader/src/loader_archetype.rs b/crates/store/re_data_loader/src/loader_archetype.rs index e1433b7f54f3..2d66e2f10166 100644 --- a/crates/store/re_data_loader/src/loader_archetype.rs +++ b/crates/store/re_data_loader/src/loader_archetype.rs @@ -1,12 +1,11 @@ use re_chunk::{Chunk, RowId}; use re_log_types::{EntityPath, TimeInt, TimePoint}; -use re_types::archetypes::VideoFrameReference; +use re_types::archetypes::{AssetVideo, VideoFrameReference}; +use re_types::components::VideoTimestamp; use re_types::Archetype; use re_types::{components::MediaType, ComponentBatch}; -use arrow2::array::{ - ListArray as ArrowListArray, NullArray as ArrowNullArray, PrimitiveArray as ArrowPrimitiveArray, -}; +use arrow2::array::PrimitiveArray as ArrowPrimitiveArray; use arrow2::Either; use crate::{DataLoader, DataLoaderError, LoadedData}; @@ -220,100 +219,60 @@ fn load_video( let video_timeline = re_log_types::Timeline::new_temporal("video"); timepoint.insert(video_timeline, re_log_types::TimeInt::new_temporal(0)); - let media_type = MediaType::guess_from_path(filepath); - - // TODO(andreas): Video frame reference generation should be available as a utility from the SDK. - - let video = if media_type.as_ref().map(|v| v.as_str()) == Some("video/mp4") { - match re_video::load_mp4(&contents) { - Ok(video) => Some(video), - Err(err) => { - re_log::warn!("Failed to load video asset {filepath:?}: {err}"); - None - } + let video_asset = AssetVideo::new(contents); + + let video_frame_reference_chunk = match video_asset.read_frame_timestamps_ns() { + Ok(frame_timestamps_ns) => { + // Time column. + let is_sorted = Some(true); + let time_column_times = ArrowPrimitiveArray::from_slice(&frame_timestamps_ns); + let time_column = + re_chunk::TimeColumn::new(is_sorted, video_timeline, time_column_times); + + // VideoTimestamp component column. + let video_timestamps = frame_timestamps_ns + .into_iter() + .map(VideoTimestamp::from_nanoseconds) + .collect::>(); + let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch; + let video_timestamp_list_array = video_timestamp_batch + .to_arrow_list_array() + .map_err(re_chunk::ChunkError::from)?; + + // Indicator column. + let video_frame_reference_indicators = + ::Indicator::new_array(video_timestamps.len()); + let video_frame_reference_indicators_list_array = video_frame_reference_indicators + .to_arrow_list_array() + .map_err(re_chunk::ChunkError::from)?; + + Some(Chunk::from_auto_row_ids( + re_chunk::ChunkId::new(), + entity_path.clone(), + std::iter::once((video_timeline, time_column)).collect(), + [ + ( + VideoFrameReference::indicator().name(), + video_frame_reference_indicators_list_array, + ), + (video_timestamp_batch.name(), video_timestamp_list_array), + ] + .into_iter() + .collect(), + )?) } - } else { - re_log::warn!("Video asset {filepath:?} has an unsupported container format."); - None - }; - // Log video frame references on the `video` timeline. - let video_frame_reference_chunk = if let Some(video) = video { - let first_timestamp = video - .segments - .first() - .map_or(0, |segment| segment.timestamp.as_nanoseconds()); - - // Time column. - let is_sorted = Some(true); - let time_column_times = - ArrowPrimitiveArray::::from_values(video.segments.iter().flat_map(|segment| { - segment - .samples - .iter() - .map(|s| s.timestamp.as_nanoseconds() - first_timestamp) - })); - - let time_column = re_chunk::TimeColumn::new(is_sorted, video_timeline, time_column_times); - - // VideoTimestamp component column. - let video_timestamps = video - .segments - .iter() - .flat_map(|segment| { - segment.samples.iter().map(|s| { - // TODO(andreas): Use sample indices instead of timestamps once possible. - re_types::components::VideoTimestamp::from_nanoseconds( - s.timestamp.as_nanoseconds(), - ) - }) - }) - .collect::>(); - let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch; - let video_timestamp_list_array = video_timestamp_batch - .to_arrow_list_array() - .map_err(re_chunk::ChunkError::from)?; - - // Indicator column. - let video_frame_reference_indicator_datatype = arrow2::datatypes::DataType::Null; - let video_frame_reference_indicator_list_array = ArrowListArray::::try_new( - ArrowListArray::::default_datatype( - video_frame_reference_indicator_datatype.clone(), - ), - video_timestamp_list_array.offsets().clone(), - Box::new(ArrowNullArray::new( - video_frame_reference_indicator_datatype, - video_timestamps.len(), - )), - None, - ) - .map_err(re_chunk::ChunkError::from)?; - - Some(Chunk::from_auto_row_ids( - re_chunk::ChunkId::new(), - entity_path.clone(), - std::iter::once((video_timeline, time_column)).collect(), - [ - ( - VideoFrameReference::indicator().name(), - video_frame_reference_indicator_list_array, - ), - (video_timestamp_batch.name(), video_timestamp_list_array), - ] - .into_iter() - .collect(), - )?) - } else { - None + Err(err) => { + re_log::warn_once!( + "Failed to read frame timestamps from video asset {filepath:?}: {err}" + ); + None + } }; // Put video asset into its own chunk since it can be fairly large. let video_asset_chunk = Chunk::builder(entity_path.clone()) - .with_archetype( - RowId::new(), - timepoint.clone(), - &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type.clone()), - ) + .with_archetype(RowId::new(), timepoint.clone(), &video_asset) .with_component_batch(RowId::new(), timepoint.clone(), &ExperimentalFeature) .build()?; diff --git a/crates/store/re_types/Cargo.toml b/crates/store/re_types/Cargo.toml index b03ded33f1d0..022912fc5e96 100644 --- a/crates/store/re_types/Cargo.toml +++ b/crates/store/re_types/Cargo.toml @@ -25,7 +25,7 @@ features = ["all"] default = ["ecolor"] ## All features except `testing`. -all = ["ecolor", "egui_plot", "glam", "image", "mint", "serde"] +all = ["ecolor", "egui_plot", "glam", "image", "mint", "serde", "video"] ## Enable color conversions. ecolor = ["dep:ecolor"] @@ -39,7 +39,7 @@ glam = ["dep:glam"] ## Integration with the [`image`](https://crates.io/crates/image/) crate, plus JPEG support. image = ["dep:ecolor", "dep:image"] -## Conversion to/from our video format +## Inspecting video data. video = ["dep:re_video"] ## Enable (de)serialization using serde. diff --git a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs index 3ad6183fbe9a..870078030b48 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs @@ -8,8 +8,9 @@ namespace rerun.archetypes; /// /// In order to display a video, you need to log a [archetypes.VideoFrameReference] for each frame. /// -/// \example archetypes/video_manual_frames title="Video with explicit frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" -// TODO(#7368): Example and reference to `send_video_frames` API. +/// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" +/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" +// TODO(#7420): update screenshot for manual frames example table AssetVideo ( "attr.docs.unreleased", "attr.rerun.experimental" diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs index 50c28176429f..606386b946ec 100644 --- a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs +++ b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs @@ -5,8 +5,9 @@ namespace rerun.archetypes; /// Used to display individual video frames from a [archetypes.AssetVideo]. /// To show an entire video, a fideo frame reference for each frame of the video should be logged. /// -/// \example archetypes/video_manual_frames title="Video with explicit frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" -// TODO(#7368): Example and reference to `send_video_frames` API. +/// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" +/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png" +// TODO(#7420): update screenshot for manual frames example table VideoFrameReference ( "attr.docs.unreleased", "attr.rerun.experimental" diff --git a/crates/store/re_types/src/archetypes/asset_video.rs b/crates/store/re_types/src/archetypes/asset_video.rs index ba29d3d91e6f..3e91d5761c48 100644 --- a/crates/store/re_types/src/archetypes/asset_video.rs +++ b/crates/store/re_types/src/archetypes/asset_video.rs @@ -28,11 +28,11 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// /// ⚠️ **This type is experimental and may be removed in future versions** /// -/// ## Example +/// ## Examples /// -/// ### Video with explicit frames +/// ### Video with automatically determined frames /// ```ignore -/// use rerun::{external::anyhow, TimeColumn}; +/// use rerun::external::anyhow; /// /// fn main() -> anyhow::Result<()> { /// let args = _args; @@ -42,31 +42,77 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// }; /// /// let rec = -/// rerun::RecordingStreamBuilder::new("rerun_example_asset_video_manual_frames").spawn()?; +/// rerun::RecordingStreamBuilder::new("rerun_example_asset_video_auto_frames").spawn()?; /// /// // Log video asset which is referred to by frame references. -/// rec.set_time_seconds("video_time", 0.0); // Make sure it's available on the timeline used for the frame references. -/// rec.log("video", &rerun::AssetVideo::from_file_path(path)?)?; +/// let video_asset = rerun::AssetVideo::from_file_path(path)?; +/// rec.log_static("video", &video_asset)?; /// -/// // Send frame references for every 0.1 seconds over a total of 10 seconds. -/// // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. -/// // TODO(#7368): Point to example using `send_video_frames`. -/// // -/// // Use `send_columns` to send all frame references in a single call. -/// let times = (0..(10 * 10)).map(|t| t as f64 * 0.1).collect::>(); -/// let time_column = TimeColumn::new_seconds("video_time", times.iter().copied()); -/// let frame_reference_indicators = -/// ::Indicator::new_array(times.len()); -/// let video_timestamps = times -/// .into_iter() -/// .map(rerun::components::VideoTimestamp::from_seconds) +/// // Send automatically determined video frame timestamps. +/// let frame_timestamps_ns = video_asset.read_frame_timestamps_ns()?; +/// let video_timestamps_ns = frame_timestamps_ns +/// .iter() +/// .copied() +/// .map(rerun::components::VideoTimestamp::from_nanoseconds) /// .collect::>(); +/// let time_column = rerun::TimeColumn::new_nanos( +/// "video_time", +/// // Note timeline values don't have to be the same as the video timestamps. +/// frame_timestamps_ns, +/// ); +/// let frame_reference_indicators = +/// ::Indicator::new_array( +/// time_column.num_rows(), +/// ); /// rec.send_columns( /// "video", /// [time_column], -/// [&frame_reference_indicators as _, &video_timestamps as _], +/// [&frame_reference_indicators as _, &video_timestamps_ns as _], +/// )?; +/// +/// Ok(()) +/// } +/// ``` +///
+/// +/// +/// +/// +/// +/// +/// +///
+/// +/// ### Demonstrates manual use of video frame references +/// ```ignore +/// use rerun::external::anyhow; +/// +/// fn main() -> anyhow::Result<()> { +/// let args = _args; +/// let Some(path) = args.get(1) else { +/// // TODO(#7354): Only mp4 is supported for now. +/// anyhow::bail!("Usage: {} ", args[0]); +/// }; +/// +/// let rec = +/// rerun::RecordingStreamBuilder::new("rerun_example_asset_video_manual_frames").spawn()?; +/// +/// // Log video asset which is referred to by frame references. +/// rec.log_static("video_asset", &rerun::AssetVideo::from_file_path(path)?)?; +/// +/// // Create two entities, showing the same video frozen at different times. +/// rec.log( +/// "frame_at_start", +/// &rerun::VideoFrameReference::new(rerun::components::VideoTimestamp::from_seconds(0.0)) +/// .with_video_reference("video_asset"), +/// )?; +/// rec.log( +/// "frame_at_one_second", +/// &rerun::VideoFrameReference::new(rerun::components::VideoTimestamp::from_seconds(1.0)) +/// .with_video_reference("video_asset"), /// )?; /// +/// // TODO(#5520): log blueprint once supported /// Ok(()) /// } /// ``` diff --git a/crates/store/re_types/src/archetypes/asset_video_ext.rs b/crates/store/re_types/src/archetypes/asset_video_ext.rs index 2d7c3074eeda..8eaf5175402c 100644 --- a/crates/store/re_types/src/archetypes/asset_video_ext.rs +++ b/crates/store/re_types/src/archetypes/asset_video_ext.rs @@ -38,4 +38,17 @@ impl AssetVideo { media_type, } } + + /// Determines the presentation timestamps of all frames inside the video. + /// + /// Returned timestamps are in nanoseconds since start and are guaranteed to be monotonically increasing. + #[cfg(feature = "video")] + pub fn read_frame_timestamps_ns(&self) -> Result, re_video::VideoLoadError> { + Ok(re_video::VideoData::load_from_bytes( + self.blob.as_slice(), + self.media_type.as_ref().map(|m| m.as_str()), + )? + .frame_timestamps_ns() + .collect()) + } } diff --git a/crates/store/re_types/src/archetypes/video_frame_reference.rs b/crates/store/re_types/src/archetypes/video_frame_reference.rs index aeca145292ea..c876ce0a8d32 100644 --- a/crates/store/re_types/src/archetypes/video_frame_reference.rs +++ b/crates/store/re_types/src/archetypes/video_frame_reference.rs @@ -25,11 +25,11 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// /// ⚠️ **This type is experimental and may be removed in future versions** /// -/// ## Example +/// ## Examples /// -/// ### Video with explicit frames +/// ### Video with automatically determined frames /// ```ignore -/// use rerun::{external::anyhow, TimeColumn}; +/// use rerun::external::anyhow; /// /// fn main() -> anyhow::Result<()> { /// let args = _args; @@ -39,31 +39,77 @@ use ::re_types_core::{DeserializationError, DeserializationResult}; /// }; /// /// let rec = -/// rerun::RecordingStreamBuilder::new("rerun_example_asset_video_manual_frames").spawn()?; +/// rerun::RecordingStreamBuilder::new("rerun_example_asset_video_auto_frames").spawn()?; /// /// // Log video asset which is referred to by frame references. -/// rec.set_time_seconds("video_time", 0.0); // Make sure it's available on the timeline used for the frame references. -/// rec.log("video", &rerun::AssetVideo::from_file_path(path)?)?; +/// let video_asset = rerun::AssetVideo::from_file_path(path)?; +/// rec.log_static("video", &video_asset)?; /// -/// // Send frame references for every 0.1 seconds over a total of 10 seconds. -/// // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. -/// // TODO(#7368): Point to example using `send_video_frames`. -/// // -/// // Use `send_columns` to send all frame references in a single call. -/// let times = (0..(10 * 10)).map(|t| t as f64 * 0.1).collect::>(); -/// let time_column = TimeColumn::new_seconds("video_time", times.iter().copied()); -/// let frame_reference_indicators = -/// ::Indicator::new_array(times.len()); -/// let video_timestamps = times -/// .into_iter() -/// .map(rerun::components::VideoTimestamp::from_seconds) +/// // Send automatically determined video frame timestamps. +/// let frame_timestamps_ns = video_asset.read_frame_timestamps_ns()?; +/// let video_timestamps_ns = frame_timestamps_ns +/// .iter() +/// .copied() +/// .map(rerun::components::VideoTimestamp::from_nanoseconds) /// .collect::>(); +/// let time_column = rerun::TimeColumn::new_nanos( +/// "video_time", +/// // Note timeline values don't have to be the same as the video timestamps. +/// frame_timestamps_ns, +/// ); +/// let frame_reference_indicators = +/// ::Indicator::new_array( +/// time_column.num_rows(), +/// ); /// rec.send_columns( /// "video", /// [time_column], -/// [&frame_reference_indicators as _, &video_timestamps as _], +/// [&frame_reference_indicators as _, &video_timestamps_ns as _], +/// )?; +/// +/// Ok(()) +/// } +/// ``` +///
+/// +/// +/// +/// +/// +/// +/// +///
+/// +/// ### Demonstrates manual use of video frame references +/// ```ignore +/// use rerun::external::anyhow; +/// +/// fn main() -> anyhow::Result<()> { +/// let args = _args; +/// let Some(path) = args.get(1) else { +/// // TODO(#7354): Only mp4 is supported for now. +/// anyhow::bail!("Usage: {} ", args[0]); +/// }; +/// +/// let rec = +/// rerun::RecordingStreamBuilder::new("rerun_example_asset_video_manual_frames").spawn()?; +/// +/// // Log video asset which is referred to by frame references. +/// rec.log_static("video_asset", &rerun::AssetVideo::from_file_path(path)?)?; +/// +/// // Create two entities, showing the same video frozen at different times. +/// rec.log( +/// "frame_at_start", +/// &rerun::VideoFrameReference::new(rerun::components::VideoTimestamp::from_seconds(0.0)) +/// .with_video_reference("video_asset"), +/// )?; +/// rec.log( +/// "frame_at_one_second", +/// &rerun::VideoFrameReference::new(rerun::components::VideoTimestamp::from_seconds(1.0)) +/// .with_video_reference("video_asset"), /// )?; /// +/// // TODO(#5520): log blueprint once supported /// Ok(()) /// } /// ``` diff --git a/crates/store/re_types/src/components/media_type_ext.rs b/crates/store/re_types/src/components/media_type_ext.rs index 834a004aa865..dc3173aa2ec7 100644 --- a/crates/store/re_types/src/components/media_type_ext.rs +++ b/crates/store/re_types/src/components/media_type_ext.rs @@ -46,6 +46,9 @@ impl MediaType { /// pub const STL: &'static str = "model/stl"; + // ------------------------------------------------------- + /// Videos: + /// [MP4 video](https://en.wikipedia.org/wiki/MP4_file_format): `video/mp4`. /// /// @@ -107,6 +110,9 @@ impl MediaType { Self(Self::STL.into()) } + // ------------------------------------------------------- + // Video: + /// `video/mp4` #[inline] pub fn mp4() -> Self { @@ -245,6 +251,7 @@ fn test_media_type_extension() { assert_eq!(MediaType::glb().file_extension(), Some("glb")); assert_eq!(MediaType::gltf().file_extension(), Some("gltf")); assert_eq!(MediaType::jpeg().file_extension(), Some("jpg")); + assert_eq!(MediaType::mp4().file_extension(), Some("mp4")); assert_eq!(MediaType::markdown().file_extension(), Some("md")); assert_eq!(MediaType::plain_text().file_extension(), Some("txt")); assert_eq!(MediaType::png().file_extension(), Some("png")); diff --git a/crates/store/re_types_core/src/datatypes/entity_path_ext.rs b/crates/store/re_types_core/src/datatypes/entity_path_ext.rs new file mode 100644 index 000000000000..babef32b7818 --- /dev/null +++ b/crates/store/re_types_core/src/datatypes/entity_path_ext.rs @@ -0,0 +1,52 @@ +use super::EntityPath; + +impl EntityPath { + #[inline] + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl From for EntityPath { + #[inline] + fn from(value: String) -> Self { + Self(value.into()) + } +} + +impl From<&str> for EntityPath { + #[inline] + fn from(value: &str) -> Self { + Self(value.into()) + } +} + +impl From for String { + #[inline] + fn from(value: EntityPath) -> Self { + value.as_str().to_owned() + } +} + +impl AsRef for EntityPath { + #[inline] + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl std::borrow::Borrow for EntityPath { + #[inline] + fn borrow(&self) -> &str { + self.as_str() + } +} + +impl std::ops::Deref for EntityPath { + type Target = str; + + #[inline] + fn deref(&self) -> &str { + self.as_str() + } +} diff --git a/crates/store/re_types_core/src/datatypes/mod.rs b/crates/store/re_types_core/src/datatypes/mod.rs index eea41d67b2f2..0ddbfbaa39e8 100644 --- a/crates/store/re_types_core/src/datatypes/mod.rs +++ b/crates/store/re_types_core/src/datatypes/mod.rs @@ -3,6 +3,7 @@ mod bool; mod bool_ext; mod entity_path; +mod entity_path_ext; mod float32; mod float32_ext; mod float64; diff --git a/crates/store/re_video/Cargo.toml b/crates/store/re_video/Cargo.toml index 24b0072ab5f3..3f1d45f98717 100644 --- a/crates/store/re_video/Cargo.toml +++ b/crates/store/re_video/Cargo.toml @@ -26,5 +26,7 @@ features = ["all"] [features] [dependencies] +itertools.workspace = true mp4.workspace = true ordered-float.workspace = true +thiserror.workspace = true diff --git a/crates/store/re_video/src/lib.rs b/crates/store/re_video/src/lib.rs index f84a836aaaf5..62204081dedc 100644 --- a/crates/store/re_video/src/lib.rs +++ b/crates/store/re_video/src/lib.rs @@ -1,9 +1,11 @@ //! Video decoding library. //! -//! The entry point is [`load_mp4`], which produces an instance of [`VideoData`]. +//! The entry point is [`VideoData::load_from_bytes`] +//! which produces an instance of [`VideoData`] from any supported video container. mod mp4; -pub use mp4::load_mp4; + +use itertools::Itertools; use ordered_float::OrderedFloat; /// Decoded video data. @@ -22,6 +24,46 @@ pub struct VideoData { pub data: Vec, } +impl VideoData { + /// Loads a video from the given data. + /// + /// TODO(andreas, jan): This should not copy the data, but instead store slices into a shared buffer. + /// at the very least the should be a way to extract only metadata. + pub fn load_from_bytes(data: &[u8], media_type: Option<&str>) -> Result { + // Media type guessing here should be identical to `re_types::MediaType::guess_from_data`, + // but we don't want to depend on `re_types` here. + let media_type = if let Some(media_type) = media_type { + media_type.to_owned() + } else if mp4::is_mp4(data) { + "video/mp4".to_owned() + } else { + // Technically this means that we failed to determine the media type altogether, + // but we don't want to call it `FailedToDetermineMediaType` since the rest of Rerun has + // access to `re_types::components::MediaType` which has a much wider range of media type detection. + return Err(VideoLoadError::UnsupportedVideoType); + }; + + match media_type.as_str() { + "video/mp4" => mp4::load_mp4(data), + media_type => Err(VideoLoadError::UnsupportedMediaType(media_type.to_owned())), + } + } + + /// Determines the presentation timestamps of all frames inside a video, returning raw time values. + /// + /// Returned timestamps are in nanoseconds since start and are guaranteed to be monotonically increasing. + pub fn frame_timestamps_ns(&self) -> impl Iterator + '_ { + // Segments are guaranteed to be sorted among each other, but within a segment, + // presentation timestamps may not be sorted since this is sorted by decode timestamps. + self.segments.iter().flat_map(|seg| { + seg.samples + .iter() + .map(|sample| sample.timestamp.as_nanoseconds()) + .sorted() + }) + } +} + /// A segment of a video. #[derive(Clone)] pub struct Segment { @@ -107,39 +149,28 @@ impl std::ops::Sub for TimeMs { } /// Errors that can occur when loading a video. -#[derive(Debug)] +#[derive(thiserror::Error, Debug)] pub enum VideoLoadError { - ParseMp4(::mp4::Error), + #[error("Failed to determine media type from data: {0}")] + ParseMp4(#[from] ::mp4::Error), + + #[error("Video file has no video tracks")] NoVideoTrack, + + #[error("Video file track config is invalid")] InvalidConfigFormat, + + #[error("Video file has invalid sample entries")] InvalidSamples, - UnsupportedMediaType(String), - UnknownMediaType, - UnsupportedCodec(String), -} -impl std::fmt::Display for VideoLoadError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::ParseMp4(err) => write!(f, "failed to parse video: {err}"), - Self::NoVideoTrack => write!(f, "video file has no video tracks"), - Self::InvalidConfigFormat => write!(f, "video file track config is invalid"), - Self::InvalidSamples => write!(f, "video file has invalid sample entries"), - Self::UnsupportedMediaType(type_) => { - write!(f, "unsupported media type {type_:?}") - } - Self::UnknownMediaType => write!(f, "unknown media type"), - Self::UnsupportedCodec(codec) => write!(f, "unsupported codec {codec:?}"), - } - } -} + #[error("Video file has unsupported media type {0}")] + UnsupportedMediaType(String), -impl std::error::Error for VideoLoadError {} + #[error("Video file has unsupported format")] + UnsupportedVideoType, -impl From<::mp4::Error> for VideoLoadError { - fn from(value: ::mp4::Error) -> Self { - Self::ParseMp4(value) - } + #[error("Video file has unsupported codec {0}")] + UnsupportedCodec(String), } impl std::fmt::Debug for VideoData { diff --git a/crates/store/re_video/src/mp4.rs b/crates/store/re_video/src/mp4.rs index b3806121765c..5d28efb91805 100644 --- a/crates/store/re_video/src/mp4.rs +++ b/crates/store/re_video/src/mp4.rs @@ -94,3 +94,39 @@ pub fn load_mp4(bytes: &[u8]) -> Result { segments, }) } + +/// Returns whether a buffer is MP4 video data. +/// +/// From `infer` crate. +pub fn is_mp4(buf: &[u8]) -> bool { + buf.len() > 11 + && (buf[4] == b'f' && buf[5] == b't' && buf[6] == b'y' && buf[7] == b'p') + && ((buf[8] == b'a' && buf[9] == b'v' && buf[10] == b'c' && buf[11] == b'1') + || (buf[8] == b'd' && buf[9] == b'a' && buf[10] == b's' && buf[11] == b'h') + || (buf[8] == b'i' && buf[9] == b's' && buf[10] == b'o' && buf[11] == b'2') + || (buf[8] == b'i' && buf[9] == b's' && buf[10] == b'o' && buf[11] == b'3') + || (buf[8] == b'i' && buf[9] == b's' && buf[10] == b'o' && buf[11] == b'4') + || (buf[8] == b'i' && buf[9] == b's' && buf[10] == b'o' && buf[11] == b'5') + || (buf[8] == b'i' && buf[9] == b's' && buf[10] == b'o' && buf[11] == b'6') + || (buf[8] == b'i' && buf[9] == b's' && buf[10] == b'o' && buf[11] == b'm') + || (buf[8] == b'm' && buf[9] == b'm' && buf[10] == b'p' && buf[11] == b'4') + || (buf[8] == b'm' && buf[9] == b'p' && buf[10] == b'4' && buf[11] == b'1') + || (buf[8] == b'm' && buf[9] == b'p' && buf[10] == b'4' && buf[11] == b'2') + || (buf[8] == b'm' && buf[9] == b'p' && buf[10] == b'4' && buf[11] == b'v') + || (buf[8] == b'm' && buf[9] == b'p' && buf[10] == b'7' && buf[11] == b'1') + || (buf[8] == b'M' && buf[9] == b'S' && buf[10] == b'N' && buf[11] == b'V') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'A' && buf[11] == b'S') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'S' && buf[11] == b'C') + || (buf[8] == b'N' && buf[9] == b'S' && buf[10] == b'D' && buf[11] == b'C') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'S' && buf[11] == b'H') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'S' && buf[11] == b'M') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'S' && buf[11] == b'P') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'S' && buf[11] == b'S') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'X' && buf[11] == b'C') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'X' && buf[11] == b'H') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'X' && buf[11] == b'M') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'X' && buf[11] == b'P') + || (buf[8] == b'N' && buf[9] == b'D' && buf[10] == b'X' && buf[11] == b'S') + || (buf[8] == b'F' && buf[9] == b'4' && buf[10] == b'V' && buf[11] == b' ') + || (buf[8] == b'F' && buf[9] == b'4' && buf[10] == b'P' && buf[11] == b' ')) +} diff --git a/crates/top/rerun_c/Cargo.toml b/crates/top/rerun_c/Cargo.toml index d1c21f196cc9..c949fce3ec04 100644 --- a/crates/top/rerun_c/Cargo.toml +++ b/crates/top/rerun_c/Cargo.toml @@ -36,6 +36,7 @@ test = false [dependencies] re_log = { workspace = true, features = ["setup"] } re_sdk = { workspace = true, features = ["data_loaders"] } +re_video.workspace = true ahash.workspace = true arrow2.workspace = true diff --git a/crates/top/rerun_c/src/lib.rs b/crates/top/rerun_c/src/lib.rs index 3cfb74edcafd..d1f8785a6baf 100644 --- a/crates/top/rerun_c/src/lib.rs +++ b/crates/top/rerun_c/src/lib.rs @@ -10,6 +10,7 @@ mod component_type_registry; mod error; mod ptr; mod recording_streams; +mod video; use std::{ collections::BTreeMap, @@ -286,6 +287,9 @@ pub enum CErrorCode { ArrowFfiSchemaImportError, ArrowFfiArrayImportError, + _CategoryUtilities = 0x0001_0000, + VideoLoadError, + Unknown = 0xFFFF_FFFF, } diff --git a/crates/top/rerun_c/src/video.rs b/crates/top/rerun_c/src/video.rs new file mode 100644 index 000000000000..c83a5e671872 --- /dev/null +++ b/crates/top/rerun_c/src/video.rs @@ -0,0 +1,51 @@ +use crate::{CError, CErrorCode, CStringView}; + +#[allow(unsafe_code)] +#[no_mangle] +pub extern "C" fn rr_video_asset_read_frame_timestamps_ns( + video_bytes: *const u8, + video_bytes_len: u64, + media_type: CStringView, + alloc_context: *mut std::ffi::c_void, + alloc_func: Option< + extern "C" fn(context: *mut std::ffi::c_void, num_timestamps: u32) -> *mut i64, + >, + error: *mut CError, +) -> *mut i64 { + if video_bytes.is_null() { + CError::unexpected_null("video_bytes").write_error(error); + return std::ptr::null_mut(); + } + let Some(alloc_func) = alloc_func else { + CError::unexpected_null("alloc_func").write_error(error); + return std::ptr::null_mut(); + }; + + let video_bytes = unsafe { std::slice::from_raw_parts(video_bytes, video_bytes_len as usize) }; + let media_type_str = media_type.as_str("media_type").ok(); + + let video = match re_video::VideoData::load_from_bytes(video_bytes, media_type_str) { + Ok(video) => video, + Err(err) => { + CError::new( + CErrorCode::VideoLoadError, + &format!("Failed to load video data: {err}"), + ) + .write_error(error); + return std::ptr::null_mut(); + } + }; + + // TODO(andreas): Producing this iterator isn't super expensive, but an ExactSizeIterator would be good to avoid + // the somewhat brittle size-oracle here! + // (note that since we create a slice from the allocation, this won't be able to go out of bound even if this value is too small) + let num_timestamps = video.segments.iter().map(|s| s.samples.len()).sum(); + let timestamps_ns_memory = alloc_func(alloc_context, num_timestamps as u32); + let timestamps_ns = + unsafe { std::slice::from_raw_parts_mut(timestamps_ns_memory, num_timestamps) }; + for (segment, timestamp_ns) in video.frame_timestamps_ns().zip(timestamps_ns.iter_mut()) { + *timestamp_ns = segment; + } + + timestamps_ns.as_mut_ptr() +} diff --git a/crates/viewer/re_renderer/src/video/mod.rs b/crates/viewer/re_renderer/src/video/mod.rs index 7b981b48fe28..0c54d5efc2e9 100644 --- a/crates/viewer/re_renderer/src/video/mod.rs +++ b/crates/viewer/re_renderer/src/video/mod.rs @@ -63,18 +63,10 @@ impl Video { /// - `video/mp4` pub fn load( render_context: &RenderContext, - media_type: Option<&str>, data: &[u8], + media_type: Option<&str>, ) -> Result { - let data = match media_type { - Some("video/mp4") => re_video::load_mp4(data)?, - Some(media_type) => { - return Err(VideoError::Load(VideoLoadError::UnsupportedMediaType( - media_type.to_owned(), - ))) - } - None => return Err(VideoError::Load(VideoLoadError::UnknownMediaType)), - }; + let data = re_video::VideoData::load_from_bytes(data, media_type)?; let decoder = decoder::VideoDecoder::new(render_context, data)?; Ok(Self { decoder }) diff --git a/crates/viewer/re_space_view_spatial/src/video_cache.rs b/crates/viewer/re_space_view_spatial/src/video_cache.rs index 8c6768dee6f4..4b197a7d5fa7 100644 --- a/crates/viewer/re_space_view_spatial/src/video_cache.rs +++ b/crates/viewer/re_space_view_spatial/src/video_cache.rs @@ -41,14 +41,14 @@ impl VideoCache { let entry = self.0.entry(key).or_insert_with(|| { re_log::debug!("Loading video {name:?}…"); - let result = Video::load(render_ctx, media_type, video_data); - let video = match result { + let video = match Video::load(render_ctx, video_data, media_type) { Ok(video) => Some(Arc::new(Mutex::new(video))), Err(err) => { re_log::warn_once!("Failed to load video {name:?}: {err}"); None } }; + Entry { used_this_frame: AtomicBool::new(false), video, diff --git a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs index b89fcebeb5d5..340e99763c69 100644 --- a/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs +++ b/crates/viewer/re_space_view_spatial/src/visualizers/videos.rs @@ -258,7 +258,7 @@ fn latest_at_query_video_from_datastore( media_type: media_type.clone(), }, &blob, - media_type.as_ref().map(|v| v.as_str()), + media_type.as_ref().map(|m| m.as_str()), ctx.render_ctx?, ) }) diff --git a/docs/content/reference/types/archetypes/asset_video.md b/docs/content/reference/types/archetypes/asset_video.md index b1cd16bda2ca..1ea3ac4780dc 100644 --- a/docs/content/reference/types/archetypes/asset_video.md +++ b/docs/content/reference/types/archetypes/asset_video.md @@ -25,9 +25,21 @@ In order to display a video, you need to log a [`archetypes.VideoFrameReference` * 🐍 [Python API docs for `AssetVideo`](https://ref.rerun.io/docs/python/stable/common/archetypes?speculative-link#rerun.archetypes.AssetVideo) * 🦀 [Rust API docs for `AssetVideo`](https://docs.rs/rerun/latest/rerun/archetypes/struct.AssetVideo.html?speculative-link) -## Example +## Examples -### Video with explicit frames +### Video with automatically determined frames + +snippet: archetypes/video_auto_frames + + + + + + + + + +### Demonstrates manual use of video frame references snippet: archetypes/video_manual_frames diff --git a/docs/content/reference/types/archetypes/video_frame_reference.md b/docs/content/reference/types/archetypes/video_frame_reference.md index 469964bfa253..535ba5019308 100644 --- a/docs/content/reference/types/archetypes/video_frame_reference.md +++ b/docs/content/reference/types/archetypes/video_frame_reference.md @@ -22,9 +22,21 @@ To show an entire video, a fideo frame reference for each frame of the video sho * 🐍 [Python API docs for `VideoFrameReference`](https://ref.rerun.io/docs/python/stable/common/archetypes?speculative-link#rerun.archetypes.VideoFrameReference) * 🦀 [Rust API docs for `VideoFrameReference`](https://docs.rs/rerun/latest/rerun/archetypes/struct.VideoFrameReference.html?speculative-link) -## Example +## Examples -### Video with explicit frames +### Video with automatically determined frames + +snippet: archetypes/video_auto_frames + + + + + + + + + +### Demonstrates manual use of video frame references snippet: archetypes/video_manual_frames diff --git a/docs/snippets/all/archetypes/video_auto_frames.cpp b/docs/snippets/all/archetypes/video_auto_frames.cpp new file mode 100644 index 000000000000..26f5a694af87 --- /dev/null +++ b/docs/snippets/all/archetypes/video_auto_frames.cpp @@ -0,0 +1,50 @@ +// Log a video asset using automatically determined frame references. +// TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. + +#include + +#include + +using namespace std::chrono_literals; + +int main(int argc, char* argv[]) { + if (argc < 2) { + // TODO(#7354): Only mp4 is supported for now. + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + + const auto path = argv[1]; + + const auto rec = rerun::RecordingStream("rerun_example_asset_video_auto_frames"); + rec.spawn().exit_on_failure(); + + // Log video asset which is referred to by frame references. + auto video_asset = rerun::AssetVideo::from_file(path).value_or_throw(); + rec.log_static("video", video_asset); + + // Send automatically determined video frame timestamps. + std::vector frame_timestamps_ns = + video_asset.read_frame_timestamps_ns().value_or_throw(); + // Note timeline values don't have to be the same as the video timestamps. + auto time_column = + rerun::TimeColumn::from_times("video_time", rerun::borrow(frame_timestamps_ns)); + + std::vector video_timestamps(frame_timestamps_ns.size()); + for (size_t i = 0; i < frame_timestamps_ns.size(); i++) { + video_timestamps[i] = rerun::components::VideoTimestamp(frame_timestamps_ns[i]); + } + auto video_frame_reference_indicators = + rerun::ComponentColumn::from_indicators( + static_cast(video_timestamps.size()) + ); + + rec.send_columns( + "video", + time_column, + { + video_frame_reference_indicators.value_or_throw(), + rerun::ComponentColumn::from_loggable(rerun::borrow(video_timestamps)).value_or_throw(), + } + ); +} diff --git a/docs/snippets/all/archetypes/video_auto_frames.py b/docs/snippets/all/archetypes/video_auto_frames.py new file mode 100644 index 000000000000..2e81fd8c42ba --- /dev/null +++ b/docs/snippets/all/archetypes/video_auto_frames.py @@ -0,0 +1,26 @@ +"""Log a video asset using automatically determined frame references.""" +# TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. + +import sys + +import rerun as rr + +if len(sys.argv) < 2: + # TODO(#7354): Only mp4 is supported for now. + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + +rr.init("rerun_example_asset_video_auto_frames", spawn=True) + +# Log video asset which is referred to by frame references. +video_asset = rr.AssetVideo(path=sys.argv[1]) +rr.log("video", video_asset, static=True) + +# Send automatically determined video frame timestamps. +frame_timestamps_ns = video_asset.read_frame_timestamps_ns() +rr.send_columns( + "video", + # Note timeline values don't have to be the same as the video timestamps. + times=[rr.TimeNanosColumn("video_time", frame_timestamps_ns)], + components=[rr.VideoFrameReference.indicator(), rr.components.VideoTimestamp.nanoseconds(frame_timestamps_ns)], +) diff --git a/docs/snippets/all/archetypes/video_auto_frames.rs b/docs/snippets/all/archetypes/video_auto_frames.rs new file mode 100644 index 000000000000..df52ccda0ef2 --- /dev/null +++ b/docs/snippets/all/archetypes/video_auto_frames.rs @@ -0,0 +1,43 @@ +//! Log a video asset using automatically determined frame references. +//! TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. + +use rerun::external::anyhow; + +fn main() -> anyhow::Result<()> { + let args = _args; + let Some(path) = args.get(1) else { + // TODO(#7354): Only mp4 is supported for now. + anyhow::bail!("Usage: {} ", args[0]); + }; + + let rec = + rerun::RecordingStreamBuilder::new("rerun_example_asset_video_auto_frames").spawn()?; + + // Log video asset which is referred to by frame references. + let video_asset = rerun::AssetVideo::from_file_path(path)?; + rec.log_static("video", &video_asset)?; + + // Send automatically determined video frame timestamps. + let frame_timestamps_ns = video_asset.read_frame_timestamps_ns()?; + let video_timestamps_ns = frame_timestamps_ns + .iter() + .copied() + .map(rerun::components::VideoTimestamp::from_nanoseconds) + .collect::>(); + let time_column = rerun::TimeColumn::new_nanos( + "video_time", + // Note timeline values don't have to be the same as the video timestamps. + frame_timestamps_ns, + ); + let frame_reference_indicators = + ::Indicator::new_array( + time_column.num_rows(), + ); + rec.send_columns( + "video", + [time_column], + [&frame_reference_indicators as _, &video_timestamps_ns as _], + )?; + + Ok(()) +} diff --git a/docs/snippets/all/archetypes/video_manual_frames.cpp b/docs/snippets/all/archetypes/video_manual_frames.cpp index b24fd12eef85..f4e02d3416a4 100644 --- a/docs/snippets/all/archetypes/video_manual_frames.cpp +++ b/docs/snippets/all/archetypes/video_manual_frames.cpp @@ -1,5 +1,6 @@ -// Log a video asset using manually created frame references. -// TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. +// Log a video asset using manually created frame references. +// TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. +// TODO(#7420): This sample doesn't render yet. #include @@ -20,31 +21,14 @@ int main(int argc, char* argv[]) { rec.spawn().exit_on_failure(); // Log video asset which is referred to by frame references. - // Make sure it's available on the timeline used for the frame references. - rec.set_time_seconds("video_time", 0.0); - rec.log("video", rerun::AssetVideo::from_file(path).value_or_throw()); - - // Send frame references for every 0.1 seconds over a total of 10 seconds. - // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. - // TODO(#7368): Point to example using `send_video_frames`. - // - // Use `send_columns` to send all frame references in a single call. - std::vector times(10 * 10); - std::vector video_timestamps(10 * 10); - for (size_t i = 0; i < times.size(); i++) { - times[i] = 100ms * i; - video_timestamps[i] = rerun::components::VideoTimestamp(times[i]); - } - auto video_frame_reference_indicators = - rerun::ComponentColumn::from_indicators( - static_cast(times.size()) - ); - rec.send_columns( - "video", - rerun::TimeColumn::from_times("video_time", rerun::borrow(times)), - { - video_frame_reference_indicators.value_or_throw(), - rerun::ComponentColumn::from_loggable(rerun::borrow(video_timestamps)).value_or_throw(), - } + rec.log_static("video_asset", rerun::AssetVideo::from_file(path).value_or_throw()); + + // Create two entities, showing the same video frozen at different times. + rec.log("frame_at_start", rerun::VideoFrameReference(0.0s).with_video_reference("video_asset")); + rec.log( + "frame_at_one_second", + rerun::VideoFrameReference(1.0s).with_video_reference("video_asset") ); + + // TODO(#5520): log blueprint once supported } diff --git a/docs/snippets/all/archetypes/video_manual_frames.py b/docs/snippets/all/archetypes/video_manual_frames.py index db69b510dd01..0dec27a11347 100644 --- a/docs/snippets/all/archetypes/video_manual_frames.py +++ b/docs/snippets/all/archetypes/video_manual_frames.py @@ -1,10 +1,11 @@ -"""Log a video asset using manually created frame references.""" +"""Manual use of individual video frame references.""" # TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. +# TODO(#7420): This sample doesn't render yet. import sys -import numpy as np import rerun as rr +import rerun.blueprint as rrb if len(sys.argv) < 2: # TODO(#7354): Only mp4 is supported for now. @@ -14,17 +15,25 @@ rr.init("rerun_example_asset_video_manual_frames", spawn=True) # Log video asset which is referred to by frame references. -rr.set_time_seconds("video_time", 0) # Make sure it's available on the timeline used for the frame references. -rr.log("video", rr.AssetVideo(path=sys.argv[1])) +rr.log("video_asset", rr.AssetVideo(path=sys.argv[1]), static=True) -# Send frame references for every 0.1 seconds over a total of 10 seconds. -# Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. -# TODO(#7368): Point to example using `send_video_frames`. -# -# Use `send_columns` to send all frame references in a single call. -times = np.arange(0.0, 10.0, 0.1) -rr.send_columns( - "video", - times=[rr.TimeSecondsColumn("video_time", times)], - components=[rr.VideoFrameReference.indicator(), rr.components.VideoTimestamp.seconds(times)], +# Create two entities, showing the same video frozen at different times. +rr.log( + "frame_at_start", + rr.VideoFrameReference( + timestamp=rr.components.VideoTimestamp(seconds=0.0), + video_reference="video_asset", + ), +) +rr.log( + "frame_at_one_second", + rr.VideoFrameReference( + timestamp=rr.components.VideoTimestamp(seconds=1.0), + video_reference="video_asset", + ), +) + +# Send blueprint that shows two 2D views next to each other. +rr.send_blueprint( + rrb.Horizontal(rrb.Spatial2DView(origin="frame_at_start"), rrb.Spatial2DView(origin="frame_at_one_second")) ) diff --git a/docs/snippets/all/archetypes/video_manual_frames.rs b/docs/snippets/all/archetypes/video_manual_frames.rs index 24e7f25fe799..447c98e848a6 100644 --- a/docs/snippets/all/archetypes/video_manual_frames.rs +++ b/docs/snippets/all/archetypes/video_manual_frames.rs @@ -1,7 +1,8 @@ //! Log a video asset using manually created frame references. //! TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. +//! TODO(#7420): This sample doesn't render yet. -use rerun::{external::anyhow, TimeColumn}; +use rerun::external::anyhow; fn main() -> anyhow::Result<()> { let args = _args; @@ -14,27 +15,20 @@ fn main() -> anyhow::Result<()> { rerun::RecordingStreamBuilder::new("rerun_example_asset_video_manual_frames").spawn()?; // Log video asset which is referred to by frame references. - rec.set_time_seconds("video_time", 0.0); // Make sure it's available on the timeline used for the frame references. - rec.log("video", &rerun::AssetVideo::from_file_path(path)?)?; + rec.log_static("video_asset", &rerun::AssetVideo::from_file_path(path)?)?; - // Send frame references for every 0.1 seconds over a total of 10 seconds. - // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. - // TODO(#7368): Point to example using `send_video_frames`. - // - // Use `send_columns` to send all frame references in a single call. - let times = (0..(10 * 10)).map(|t| t as f64 * 0.1).collect::>(); - let time_column = TimeColumn::new_seconds("video_time", times.iter().copied()); - let frame_reference_indicators = - ::Indicator::new_array(times.len()); - let video_timestamps = times - .into_iter() - .map(rerun::components::VideoTimestamp::from_seconds) - .collect::>(); - rec.send_columns( - "video", - [time_column], - [&frame_reference_indicators as _, &video_timestamps as _], + // Create two entities, showing the same video frozen at different times. + rec.log( + "frame_at_start", + &rerun::VideoFrameReference::new(rerun::components::VideoTimestamp::from_seconds(0.0)) + .with_video_reference("video_asset"), + )?; + rec.log( + "frame_at_one_second", + &rerun::VideoFrameReference::new(rerun::components::VideoTimestamp::from_seconds(1.0)) + .with_video_reference("video_asset"), )?; + // TODO(#5520): log blueprint once supported Ok(()) } diff --git a/docs/snippets/snippets.toml b/docs/snippets/snippets.toml index 2b74b66a7584..82ad3289c7b2 100644 --- a/docs/snippets/snippets.toml +++ b/docs/snippets/snippets.toml @@ -181,7 +181,7 @@ quick_start = [ # These examples don't have exactly the same implementation. "py", "rust", ] -"archetypes/video_manual_frames" = [ # This mixes `log` and `send_columns`. Since `log` is suspect to delays by the batcher, this test gets flaky. +"archetypes/video_auto_frames" = [ # This mixes `log` and `send_columns`. Since `log` is suspect to delays by the batcher, this test gets flaky. "cpp", "py", "rust", @@ -192,6 +192,10 @@ quick_start = [ # These examples don't have exactly the same implementation. [extra_args] "archetypes/asset3d_simple" = ["$config_dir/../../tests/assets/cube.glb"] "archetypes/asset3d_out_of_tree" = ["$config_dir/../../tests/assets/cube.glb"] +"archetypes/video_auto_frames" = [ + "$config_dir/../../tests/assets/video/Big_Buck_Bunny_1080_10s_av1.mp4", +] +# TODO(#7420): This sample doesn't render yet. Once it does it would be nice to have a video that looks significantly different after 1s. "archetypes/video_manual_frames" = [ "$config_dir/../../tests/assets/video/Big_Buck_Bunny_1080_10s_av1.mp4", ] diff --git a/rerun_cpp/src/rerun/archetypes/asset_video.hpp b/rerun_cpp/src/rerun/archetypes/asset_video.hpp index bc25a1247fe7..a39ef4208f49 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video.hpp @@ -11,6 +11,7 @@ #include "../indicator_component.hpp" #include "../result.hpp" +#include #include #include #include @@ -26,9 +27,9 @@ namespace rerun::archetypes { /// /// In order to display a video, you need to log a `archetypes::VideoFrameReference` for each frame. /// - /// ## Example + /// ## Examples /// - /// ### Video with explicit frames + /// ### Video with automatically determined frames /// ![image](https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/full.png) /// /// ```cpp @@ -47,32 +48,32 @@ namespace rerun::archetypes { /// /// const auto path = argv[1]; /// - /// const auto rec = rerun::RecordingStream("rerun_example_asset_video_manual_frames"); + /// const auto rec = rerun::RecordingStream("rerun_example_asset_video_auto_frames"); /// rec.spawn().exit_on_failure(); /// /// // Log video asset which is referred to by frame references. - /// // Make sure it's available on the timeline used for the frame references. - /// rec.set_time_seconds("video_time", 0.0); - /// rec.log("video", rerun::AssetVideo::from_file(path).value_or_throw()); - /// - /// // Send frame references for every 0.1 seconds over a total of 10 seconds. - /// // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. - /// // TODO(#7368): Point to example using `send_video_frames`. - /// // - /// // Use `send_columns` to send all frame references in a single call. - /// std::vector times(10 * 10); - /// std::vector video_timestamps(10 * 10); - /// for (size_t i = 0; i frame_timestamps_ns = + /// video_asset.read_frame_timestamps_ns().value_or_throw(); + /// // Note timeline values don't have to be the same as the video timestamps. + /// auto time_column = + /// rerun::TimeColumn::from_times("video_time", rerun::borrow(frame_timestamps_ns)); + /// + /// std::vector video_timestamps(frame_timestamps_ns.size()); + /// for (size_t i = 0; i ( - /// static_cast(times.size()) + /// static_cast(video_timestamps.size()) /// ); + /// /// rec.send_columns( /// "video", - /// rerun::TimeColumn::from_times("video_time", rerun::borrow(times)), + /// time_column, /// { /// video_frame_reference_indicators.value_or_throw(), /// rerun::ComponentColumn::from_loggable(rerun::borrow(video_timestamps)).value_or_throw(), @@ -81,6 +82,42 @@ namespace rerun::archetypes { /// } /// ``` /// + /// ### Demonstrates manual use of video frame references + /// ![image](https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/full.png) + /// + /// ```cpp + /// #include + /// + /// #include + /// + /// using namespace std::chrono_literals; + /// + /// int main(int argc, char* argv[]) { + /// if (argc <2) { + /// // TODO(#7354): Only mp4 is supported for now. + /// std::cerr <<"Usage: " <" <> read_frame_timestamps_ns() const; + // END of extensions from asset_video_ext.cpp, start of generated code: public: diff --git a/rerun_cpp/src/rerun/archetypes/asset_video_ext.cpp b/rerun_cpp/src/rerun/archetypes/asset_video_ext.cpp index 5b44f36da89c..7225dcc8acc5 100644 --- a/rerun_cpp/src/rerun/archetypes/asset_video_ext.cpp +++ b/rerun_cpp/src/rerun/archetypes/asset_video_ext.cpp @@ -2,12 +2,15 @@ #include #include +#include "../c/rerun.h" +#include "../string_utils.hpp" #include "asset_video.hpp" // It's undefined behavior to pre-declare std types, see http://www.gotw.ca/gotw/034.htm // We want to use `std::filesystem::path`, so we have it include it in the header. // +#include #include // @@ -38,6 +41,11 @@ namespace rerun::archetypes { return asset; } + /// Determines the presentation timestamps of all frames inside the video. + /// + /// Returned timestamps are in nanoseconds since start and are guaranteed to be monotonically increasing. + Result> read_frame_timestamps_ns() const; + // #endif @@ -59,4 +67,38 @@ namespace rerun::archetypes { rerun::components::MediaType::guess_from_path(path) ); } + + static int64_t* alloc_timestamps(void* alloc_context, uint32_t num_timestamps) { + auto frame_timestamps_ptr = + static_cast*>(alloc_context); + frame_timestamps_ptr->resize(num_timestamps); + return reinterpret_cast(frame_timestamps_ptr->data()); + } + + Result> AssetVideo::read_frame_timestamps_ns() const { + static_assert(sizeof(int64_t) == sizeof(std::chrono::nanoseconds::rep)); + + rr_string media_type_c = detail::to_rr_string(std::nullopt); + if (media_type.has_value()) { + media_type_c = detail::to_rr_string(media_type.value().value.value); + } + + std::vector frame_timestamps; + + rr_error status = {}; + rr_video_asset_read_frame_timestamps_ns( + blob.data.data.begin(), + blob.data.data.size(), + media_type_c, + &frame_timestamps, + &alloc_timestamps, + &status + ); + if (status.code != RR_ERROR_CODE_OK) { + return Error(status); + } + + return frame_timestamps; + } + } // namespace rerun::archetypes diff --git a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp index 814baed5c996..86e40596f1b2 100644 --- a/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp +++ b/rerun_cpp/src/rerun/archetypes/video_frame_reference.hpp @@ -22,9 +22,9 @@ namespace rerun::archetypes { /// Used to display individual video frames from a `archetypes::AssetVideo`. /// To show an entire video, a fideo frame reference for each frame of the video should be logged. /// - /// ## Example + /// ## Examples /// - /// ### Video with explicit frames + /// ### Video with automatically determined frames /// ![image](https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/full.png) /// /// ```cpp @@ -43,32 +43,32 @@ namespace rerun::archetypes { /// /// const auto path = argv[1]; /// - /// const auto rec = rerun::RecordingStream("rerun_example_asset_video_manual_frames"); + /// const auto rec = rerun::RecordingStream("rerun_example_asset_video_auto_frames"); /// rec.spawn().exit_on_failure(); /// /// // Log video asset which is referred to by frame references. - /// // Make sure it's available on the timeline used for the frame references. - /// rec.set_time_seconds("video_time", 0.0); - /// rec.log("video", rerun::AssetVideo::from_file(path).value_or_throw()); - /// - /// // Send frame references for every 0.1 seconds over a total of 10 seconds. - /// // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. - /// // TODO(#7368): Point to example using `send_video_frames`. - /// // - /// // Use `send_columns` to send all frame references in a single call. - /// std::vector times(10 * 10); - /// std::vector video_timestamps(10 * 10); - /// for (size_t i = 0; i frame_timestamps_ns = + /// video_asset.read_frame_timestamps_ns().value_or_throw(); + /// // Note timeline values don't have to be the same as the video timestamps. + /// auto time_column = + /// rerun::TimeColumn::from_times("video_time", rerun::borrow(frame_timestamps_ns)); + /// + /// std::vector video_timestamps(frame_timestamps_ns.size()); + /// for (size_t i = 0; i ( - /// static_cast(times.size()) + /// static_cast(video_timestamps.size()) /// ); + /// /// rec.send_columns( /// "video", - /// rerun::TimeColumn::from_times("video_time", rerun::borrow(times)), + /// time_column, /// { /// video_frame_reference_indicators.value_or_throw(), /// rerun::ComponentColumn::from_loggable(rerun::borrow(video_timestamps)).value_or_throw(), @@ -77,6 +77,42 @@ namespace rerun::archetypes { /// } /// ``` /// + /// ### Demonstrates manual use of video frame references + /// ![image](https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/full.png) + /// + /// ```cpp + /// #include + /// + /// #include + /// + /// using namespace std::chrono_literals; + /// + /// int main(int argc, char* argv[]) { + /// if (argc <2) { + /// // TODO(#7354): Only mp4 is supported for now. + /// std::cerr <<"Usage: " <" < + EntityPath(std::string_view path_) : value(std::string(path_)) {} + + EntityPath(const char* path_) : value(std::string(path_)) {} + // + +} // namespace rerun::components +#endif diff --git a/rerun_cpp/src/rerun/components/media_type.hpp b/rerun_cpp/src/rerun/components/media_type.hpp index 1c154a41de5a..df9ea1a5aeeb 100644 --- a/rerun_cpp/src/rerun/components/media_type.hpp +++ b/rerun_cpp/src/rerun/components/media_type.hpp @@ -86,6 +86,9 @@ namespace rerun::components { return "model/stl"; } + // ------------------------------------------------------- + /// Videos: + /// [MP4 video](https://en.wikipedia.org/wiki/MP4_file_format): `video/mp4`. /// /// diff --git a/rerun_cpp/src/rerun/components/media_type_ext.cpp b/rerun_cpp/src/rerun/components/media_type_ext.cpp index 2c402b4986bc..968c5b1ca7db 100644 --- a/rerun_cpp/src/rerun/components/media_type_ext.cpp +++ b/rerun_cpp/src/rerun/components/media_type_ext.cpp @@ -90,6 +90,9 @@ namespace rerun { return "model/stl"; } + // ------------------------------------------------------- + /// Videos: + /// [MP4 video](https://en.wikipedia.org/wiki/MP4_file_format): `video/mp4`. /// /// diff --git a/rerun_cpp/src/rerun/error.hpp b/rerun_cpp/src/rerun/error.hpp index ce4838faa22a..979b49324910 100644 --- a/rerun_cpp/src/rerun/error.hpp +++ b/rerun_cpp/src/rerun/error.hpp @@ -57,8 +57,12 @@ namespace rerun { ArrowFfiSchemaImportError, ArrowFfiArrayImportError, + // Utility errors. + _CategoryUtilities = 0x0001'0000, + VideoLoadError, + // Errors relating to file IO. - _CategoryFileIO = 0x0001'0000, + _CategoryFileIO = 0x0010'0000, FileOpenFailure, // Errors directly translated from arrow::StatusCode. diff --git a/rerun_py/Cargo.toml b/rerun_py/Cargo.toml index e43151c47f2a..81e276a8fd28 100644 --- a/rerun_py/Cargo.toml +++ b/rerun_py/Cargo.toml @@ -46,6 +46,7 @@ re_log = { workspace = true, features = ["setup"] } re_log_types.workspace = true re_memory.workspace = true re_sdk = { workspace = true, features = ["data_loaders"] } +re_video.workspace = true re_web_viewer_server = { workspace = true, optional = true } re_ws_comms = { workspace = true, optional = true } diff --git a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py index 05a7d45aee47..c35d9b4ed86e 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/asset_video.py @@ -29,15 +29,14 @@ class AssetVideo(AssetVideoExt, Archetype): ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** - Example - ------- - ### Video with explicit frames: + Examples + -------- + ### Video with automatically determined frames: ```python # TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. import sys - import numpy as np import rerun as rr if len(sys.argv) < 2: @@ -45,22 +44,70 @@ class AssetVideo(AssetVideoExt, Archetype): print(f"Usage: {sys.argv[0]} ") sys.exit(1) - rr.init("rerun_example_asset_video_manual_frames", spawn=True) + rr.init("rerun_example_asset_video_auto_frames", spawn=True) # Log video asset which is referred to by frame references. - rr.set_time_seconds("video_time", 0) # Make sure it's available on the timeline used for the frame references. - rr.log("video", rr.AssetVideo(path=sys.argv[1])) + video_asset = rr.AssetVideo(path=sys.argv[1]) + rr.log("video", video_asset, static=True) - # Send frame references for every 0.1 seconds over a total of 10 seconds. - # Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. - # TODO(#7368): Point to example using `send_video_frames`. - # - # Use `send_columns` to send all frame references in a single call. - times = np.arange(0.0, 10.0, 0.1) + # Send automatically determined video frame timestamps. + frame_timestamps_ns = video_asset.read_frame_timestamps_ns() rr.send_columns( "video", - times=[rr.TimeSecondsColumn("video_time", times)], - components=[rr.VideoFrameReference.indicator(), rr.components.VideoTimestamp.seconds(times)], + # Note timeline values don't have to be the same as the video timestamps. + times=[rr.TimeNanosColumn("video_time", frame_timestamps_ns)], + components=[rr.VideoFrameReference.indicator(), rr.components.VideoTimestamp.nanoseconds(frame_timestamps_ns)], + ) + ``` +
+ + + + + + + +
+ + ### Demonstrates manual use of video frame references: + ```python + # TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. + # TODO(#7420): This sample doesn't render yet. + + import sys + + import rerun as rr + import rerun.blueprint as rrb + + if len(sys.argv) < 2: + # TODO(#7354): Only mp4 is supported for now. + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + rr.init("rerun_example_asset_video_manual_frames", spawn=True) + + # Log video asset which is referred to by frame references. + rr.log("video_asset", rr.AssetVideo(path=sys.argv[1]), static=True) + + # Create two entities, showing the same video frozen at different times. + rr.log( + "frame_at_start", + rr.VideoFrameReference( + timestamp=rr.components.VideoTimestamp(seconds=0.0), + video_reference="video_asset", + ), + ) + rr.log( + "frame_at_one_second", + rr.VideoFrameReference( + timestamp=rr.components.VideoTimestamp(seconds=1.0), + video_reference="video_asset", + ), + ) + + # Send blueprint that shows two 2D views next to each other. + rr.send_blueprint( + rrb.Horizontal(rrb.Spatial2DView(origin="frame_at_start"), rrb.Spatial2DView(origin="frame_at_one_second")) ) ```
diff --git a/rerun_py/rerun_sdk/rerun/archetypes/asset_video_ext.py b/rerun_py/rerun_sdk/rerun/archetypes/asset_video_ext.py index 23cc56fb660a..bb927c515918 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/asset_video_ext.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/asset_video_ext.py @@ -3,6 +3,10 @@ import pathlib from typing import Any +import numpy as np +import numpy.typing as npt +import rerun_bindings as bindings + from .. import datatypes from ..error_utils import catch_and_log_exceptions @@ -59,3 +63,19 @@ def __init__( return self.__attrs_clear__() + + def read_frame_timestamps_ns(self: Any) -> npt.NDArray[np.int64]: + """ + Determines the presentation timestamps of all frames inside the video. + + Throws a runtime exception if the video cannot be read. + """ + if self.blob is not None: + video_buffer = self.blob.as_arrow_array() + else: + raise RuntimeError("Asset video has no video buffer") + + if self.media_type is not None: + media_type = self.media_type.as_arrow_array().storage[0].as_py() + + return np.array(bindings.asset_video_read_frame_timestamps_ns(video_buffer, media_type), dtype=np.int64) diff --git a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py index 6dba6a947778..1b72bf645919 100644 --- a/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py +++ b/rerun_py/rerun_sdk/rerun/archetypes/video_frame_reference.py @@ -28,15 +28,14 @@ class VideoFrameReference(Archetype): ⚠️ **This is an experimental API! It is not fully supported, and is likely to change significantly in future versions.** - Example - ------- - ### Video with explicit frames: + Examples + -------- + ### Video with automatically determined frames: ```python # TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. import sys - import numpy as np import rerun as rr if len(sys.argv) < 2: @@ -44,22 +43,70 @@ class VideoFrameReference(Archetype): print(f"Usage: {sys.argv[0]} ") sys.exit(1) - rr.init("rerun_example_asset_video_manual_frames", spawn=True) + rr.init("rerun_example_asset_video_auto_frames", spawn=True) # Log video asset which is referred to by frame references. - rr.set_time_seconds("video_time", 0) # Make sure it's available on the timeline used for the frame references. - rr.log("video", rr.AssetVideo(path=sys.argv[1])) + video_asset = rr.AssetVideo(path=sys.argv[1]) + rr.log("video", video_asset, static=True) - # Send frame references for every 0.1 seconds over a total of 10 seconds. - # Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. - # TODO(#7368): Point to example using `send_video_frames`. - # - # Use `send_columns` to send all frame references in a single call. - times = np.arange(0.0, 10.0, 0.1) + # Send automatically determined video frame timestamps. + frame_timestamps_ns = video_asset.read_frame_timestamps_ns() rr.send_columns( "video", - times=[rr.TimeSecondsColumn("video_time", times)], - components=[rr.VideoFrameReference.indicator(), rr.components.VideoTimestamp.seconds(times)], + # Note timeline values don't have to be the same as the video timestamps. + times=[rr.TimeNanosColumn("video_time", frame_timestamps_ns)], + components=[rr.VideoFrameReference.indicator(), rr.components.VideoTimestamp.nanoseconds(frame_timestamps_ns)], + ) + ``` +
+ + + + + + + +
+ + ### Demonstrates manual use of video frame references: + ```python + # TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. + # TODO(#7420): This sample doesn't render yet. + + import sys + + import rerun as rr + import rerun.blueprint as rrb + + if len(sys.argv) < 2: + # TODO(#7354): Only mp4 is supported for now. + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + + rr.init("rerun_example_asset_video_manual_frames", spawn=True) + + # Log video asset which is referred to by frame references. + rr.log("video_asset", rr.AssetVideo(path=sys.argv[1]), static=True) + + # Create two entities, showing the same video frozen at different times. + rr.log( + "frame_at_start", + rr.VideoFrameReference( + timestamp=rr.components.VideoTimestamp(seconds=0.0), + video_reference="video_asset", + ), + ) + rr.log( + "frame_at_one_second", + rr.VideoFrameReference( + timestamp=rr.components.VideoTimestamp(seconds=1.0), + video_reference="video_asset", + ), + ) + + # Send blueprint that shows two 2D views next to each other. + rr.send_blueprint( + rrb.Horizontal(rrb.Spatial2DView(origin="frame_at_start"), rrb.Spatial2DView(origin="frame_at_one_second")) ) ```
diff --git a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py index 23b11eae4ddc..41fa33bcf3c0 100644 --- a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py +++ b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py @@ -23,7 +23,7 @@ class VideoTimestamp(VideoTimestampExt, datatypes.VideoTimestamp, ComponentMixin """ _BATCH_TYPE = None - # You can define your own __init__ function as a member of VideoTimestampExt in video_timestamp_ext.py + # __init__ can be found in video_timestamp_ext.py # Note: there are no fields here because VideoTimestamp delegates to datatypes.VideoTimestamp pass diff --git a/rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py b/rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py index 98403aa558cc..307ba0cb1122 100644 --- a/rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py +++ b/rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py @@ -1,18 +1,51 @@ from __future__ import annotations +from typing import Any, Union + import numpy as np import numpy.typing as npt +from rerun.error_utils import catch_and_log_exceptions + from .. import components, datatypes class VideoTimestampExt: """Extension for [VideoTimestamp][rerun.components.VideoTimestamp].""" - # Implementation note: - # We could add an init method that deals with seconds/milliseconds/nanoseconds etc. - # However, this would require _a lot_ of slow parameter validation on a per timestamp basis. - # When in actuallity, this data practically always comes in homogeneous batches. + def __init__( + self: Any, + *, + video_time: Union[int, None] = None, + time_mode: Union[datatypes.VideoTimeModeLike, None] = None, + seconds: Union[float, None] = None, + ): + """ + Create a new instance of the VideoTimestamp component. + + Parameters + ---------- + video_time: + Timestamp value, type defined by `time_mode`. + time_mode: + How to interpret `video_time`. + seconds: + The timestamp in seconds since the start of the video. + Mutually exclusive with `video_time` and `time_mode`. + + """ + + with catch_and_log_exceptions(context=self.__class__.__name__): + if seconds is not None: + if video_time is not None or time_mode is not None: + raise ValueError("Cannot specify both `seconds` and `video_time`/`time_mode`.") + video_time = int(seconds * 1e9) + time_mode = datatypes.VideoTimeMode.Nanoseconds + + self.__attrs_init__(video_time=video_time, time_mode=time_mode) + return + + self.__attrs_clear__() @staticmethod def seconds( diff --git a/rerun_py/src/arrow.rs b/rerun_py/src/arrow.rs index dde1ce3cbbdc..4adb8e710c12 100644 --- a/rerun_py/src/arrow.rs +++ b/rerun_py/src/arrow.rs @@ -24,7 +24,10 @@ use re_sdk::{ComponentName, EntityPath, Timeline}; /// Perform conversion between a pyarrow array to arrow2 types. /// /// `name` is the name of the Rerun component, and the name of the pyarrow `Field` (column name). -fn array_to_rust(arrow_array: &Bound<'_, PyAny>, name: &str) -> PyResult<(Box, Field)> { +pub fn array_to_rust( + arrow_array: &Bound<'_, PyAny>, + name: &str, +) -> PyResult<(Box, Field)> { let py_array: PyArrowType = arrow_array.extract()?; let arr1_array = make_array(py_array.0); diff --git a/rerun_py/src/lib.rs b/rerun_py/src/lib.rs index 73c8e4a368a5..cd625e1f89b4 100644 --- a/rerun_py/src/lib.rs +++ b/rerun_py/src/lib.rs @@ -15,3 +15,4 @@ static GLOBAL: AccountingAllocator = mod arrow; mod python_bridge; +mod video; diff --git a/rerun_py/src/python_bridge.rs b/rerun_py/src/python_bridge.rs index 15b472e01dce..ad194790371f 100644 --- a/rerun_py/src/python_bridge.rs +++ b/rerun_py/src/python_bridge.rs @@ -1,6 +1,6 @@ -#![allow(clippy::needless_pass_by_value)] // A lot of arguments to #[pufunction] need to be by value -#![allow(clippy::borrow_deref_ref)] // False positive due to #[pufunction] macro -#![allow(unsafe_op_in_unsafe_fn)] // False positive due to #[pufunction] macro +#![allow(clippy::needless_pass_by_value)] // A lot of arguments to #[pyfunction] need to be by value +#![allow(clippy::borrow_deref_ref)] // False positive due to #[pyfunction] macro +#![allow(unsafe_op_in_unsafe_fn)] // False positive due to #[pyfunction] macro use std::collections::HashMap; use std::io::IsTerminal as _; @@ -169,6 +169,9 @@ fn rerun_bindings(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(escape_entity_path_part, m)?)?; m.add_function(wrap_pyfunction!(new_entity_path, m)?)?; + use crate::video::asset_video_read_frame_timestamps_ns; + m.add_function(wrap_pyfunction!(asset_video_read_frame_timestamps_ns, m)?)?; + Ok(()) } diff --git a/rerun_py/src/video.rs b/rerun_py/src/video.rs new file mode 100644 index 000000000000..c0b6a9ddd01a --- /dev/null +++ b/rerun_py/src/video.rs @@ -0,0 +1,39 @@ +#![allow(unsafe_op_in_unsafe_fn)] // False positive due to #[pyfunction] macro + +use pyo3::{exceptions::PyRuntimeError, pyfunction, Bound, PyAny, PyResult}; + +use crate::arrow::array_to_rust; + +/// Reads the timestamps of all frames in a video asset. +/// +/// Implementation note: +/// On the Python side we start out with a pyarrow array of bytes. Converting it to +/// Python `bytes` can be done with `to_pybytes` but this requires copying the data. +/// So instead, we pass the arrow array directly. +#[pyfunction] +pub fn asset_video_read_frame_timestamps_ns( + video_bytes_arrow_array: &Bound<'_, PyAny>, + media_type: Option<&str>, +) -> PyResult> { + let video_bytes_arrow_array = + array_to_rust(video_bytes_arrow_array, "rerun.components.Blob")?.0; + + let video_bytes_arrow_uint8_array = video_bytes_arrow_array + .as_any() + .downcast_ref::>() + .and_then(|arr| arr.values().as_any().downcast_ref::()) + .ok_or_else(|| { + PyRuntimeError::new_err(format!( + "Expected arrow array to be a list with a single uint8 array, instead it has the datatype {:?}", + video_bytes_arrow_array.data_type() + )) + })?; + + Ok(re_video::VideoData::load_from_bytes( + video_bytes_arrow_uint8_array.values().as_slice(), + media_type, + ) + .map_err(|err| PyRuntimeError::new_err(err.to_string()))? + .frame_timestamps_ns() + .collect()) +} diff --git a/tests/python/release_checklist/check_all_components_ui.py b/tests/python/release_checklist/check_all_components_ui.py index a7daab5b2fbc..af638f024364 100644 --- a/tests/python/release_checklist/check_all_components_ui.py +++ b/tests/python/release_checklist/check_all_components_ui.py @@ -218,7 +218,7 @@ def alternatives(self) -> list[Any] | None: "TriangleIndicesBatch": TestCase(batch=[(0, 1, 2), (3, 4, 5), (6, 7, 8)]), "Vector2DBatch": TestCase(batch=[(0, 1), (2, 3), (4, 5)]), "Vector3DBatch": TestCase(batch=[(0, 3, 4), (1, 4, 5), (2, 5, 6)]), - "VideoTimestampBatch": TestCase(rr.components.VideoTimestamp(0, rr.datatypes.VideoTimeMode.Nanoseconds)), + "VideoTimestampBatch": TestCase(rr.components.VideoTimestamp(seconds=0.0)), "ViewCoordinatesBatch": TestCase(rr.components.ViewCoordinates.LBD), "VisualizerOverridesBatch": TestCase(disabled=True), # no Python-based serialization }