From 148f9fc6fdf4cdd7090c15645397a148b0413287 Mon Sep 17 00:00:00 2001 From: Andreas Reich Date: Wed, 11 Sep 2024 19:09:28 +0200 Subject: [PATCH] Add examples for manual logging of video reference + needed extensions etc. --- .../re_data_loader/src/loader_archetype.rs | 2 +- .../src/components/video_timestamp_ext.rs | 16 ++++- .../src/datatypes/video_timestamp_ext.rs | 2 +- crates/store/re_types_core/src/archetype.rs | 43 ++++++++++++ .../all/archetypes/asset3d_simple.cpp | 2 - .../all/archetypes/video_manual_frames.cpp | 47 ++++++++++++++ .../all/archetypes/video_manual_frames.py | 33 ++++++++++ .../all/archetypes/video_manual_frames.rs | 40 ++++++++++++ docs/snippets/all/archetypes/video_simple.cpp | 21 ------ docs/snippets/all/archetypes/video_simple.py | 13 ---- docs/snippets/all/archetypes/video_simple.rs | 16 ----- docs/snippets/snippets.toml | 4 +- rerun_cpp/src/rerun/archetypes/asset3d.hpp | 2 - .../src/rerun/components/video_timestamp.hpp | 34 ++++++++++ .../rerun/components/video_timestamp_ext.cpp | 50 ++++++++++++++ rerun_py/rerun_sdk/rerun/__init__.py | 1 + .../rerun/components/video_timestamp.py | 3 +- .../rerun/components/video_timestamp_ext.py | 65 +++++++++++++++++++ 18 files changed, 334 insertions(+), 60 deletions(-) create mode 100644 docs/snippets/all/archetypes/video_manual_frames.cpp create mode 100644 docs/snippets/all/archetypes/video_manual_frames.py create mode 100644 docs/snippets/all/archetypes/video_manual_frames.rs delete mode 100644 docs/snippets/all/archetypes/video_simple.cpp delete mode 100755 docs/snippets/all/archetypes/video_simple.py delete mode 100644 docs/snippets/all/archetypes/video_simple.rs create mode 100644 rerun_cpp/src/rerun/components/video_timestamp_ext.cpp create mode 100644 rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py diff --git a/crates/store/re_data_loader/src/loader_archetype.rs b/crates/store/re_data_loader/src/loader_archetype.rs index 6f29163ef7b65..e1433b7f54f3a 100644 --- a/crates/store/re_data_loader/src/loader_archetype.rs +++ b/crates/store/re_data_loader/src/loader_archetype.rs @@ -263,7 +263,7 @@ fn load_video( .flat_map(|segment| { segment.samples.iter().map(|s| { // TODO(andreas): Use sample indices instead of timestamps once possible. - re_types::components::VideoTimestamp::new_nanoseconds( + re_types::components::VideoTimestamp::from_nanoseconds( s.timestamp.as_nanoseconds(), ) }) diff --git a/crates/store/re_types/src/components/video_timestamp_ext.rs b/crates/store/re_types/src/components/video_timestamp_ext.rs index df7a6f9947bcc..48efa5617d5e5 100644 --- a/crates/store/re_types/src/components/video_timestamp_ext.rs +++ b/crates/store/re_types/src/components/video_timestamp_ext.rs @@ -1,9 +1,21 @@ use super::VideoTimestamp; impl VideoTimestamp { + /// Create new timestamp from seconds since video start. + #[inline] + pub fn from_seconds(seconds: f64) -> Self { + crate::datatypes::VideoTimestamp::from_nanoseconds((seconds * 1e9) as i64).into() + } + + /// Create new timestamp from milliseconds since video start. + #[inline] + pub fn from_milliseconds(milliseconds: f64) -> Self { + crate::datatypes::VideoTimestamp::from_nanoseconds((milliseconds * 1e6) as i64).into() + } + /// Create new timestamp from nanoseconds since video start. #[inline] - pub fn new_nanoseconds(nanos: i64) -> Self { - crate::datatypes::VideoTimestamp::new_nanoseconds(nanos).into() + pub fn from_nanoseconds(nanos: i64) -> Self { + crate::datatypes::VideoTimestamp::from_nanoseconds(nanos).into() } } diff --git a/crates/store/re_types/src/datatypes/video_timestamp_ext.rs b/crates/store/re_types/src/datatypes/video_timestamp_ext.rs index 342e0d1051db1..4e17ee943174c 100644 --- a/crates/store/re_types/src/datatypes/video_timestamp_ext.rs +++ b/crates/store/re_types/src/datatypes/video_timestamp_ext.rs @@ -3,7 +3,7 @@ use super::{VideoTimeMode, VideoTimestamp}; impl VideoTimestamp { /// Create new timestamp from nanoseconds since video start. #[inline] - pub fn new_nanoseconds(nanos: i64) -> Self { + pub fn from_nanoseconds(nanos: i64) -> Self { Self { video_time: nanos, time_mode: VideoTimeMode::Nanoseconds, diff --git a/crates/store/re_types_core/src/archetype.rs b/crates/store/re_types_core/src/archetype.rs index d00e2792f0d6c..6eef97eb81764 100644 --- a/crates/store/re_types_core/src/archetype.rs +++ b/crates/store/re_types_core/src/archetype.rs @@ -194,6 +194,18 @@ impl GenericIndicatorComponent { pub const DEFAULT: Self = Self { _phantom: std::marker::PhantomData::, }; + + /// Create an array of indicator components of this type with the given length. + /// + /// This can be useful when sending columns of indicators with + /// `rerun::RecordingStream::send_columns`. + #[inline] + pub fn new_array(len: usize) -> GenericIndicatorComponentArray { + GenericIndicatorComponentArray { + len, + _phantom: std::marker::PhantomData::, + } + } } impl Default for GenericIndicatorComponent { @@ -221,6 +233,37 @@ impl crate::LoggableBatch for GenericIndicatorComponent { impl crate::ComponentBatch for GenericIndicatorComponent {} +/// A generic [indicator component] array of a given length. +/// +/// This can be useful when sending columns of indicators with +/// `rerun::RecordingStream::send_columns`. +/// +/// To create this type, call [`GenericIndicatorComponent::new_array`]. +/// +/// [indicator component]: [`Archetype::Indicator`] +#[derive(Debug, Clone, Copy)] +pub struct GenericIndicatorComponentArray { + len: usize, + _phantom: std::marker::PhantomData, +} + +impl crate::LoggableBatch for GenericIndicatorComponentArray { + type Name = ComponentName; + + #[inline] + fn name(&self) -> Self::Name { + GenericIndicatorComponent::::DEFAULT.name() + } + + #[inline] + fn to_arrow(&self) -> SerializationResult> { + let datatype = arrow2::datatypes::DataType::Null; + Ok(arrow2::array::NullArray::new(datatype, self.len).boxed()) + } +} + +impl crate::ComponentBatch for GenericIndicatorComponentArray {} + // --- /// An arbitrary named [indicator component]. diff --git a/docs/snippets/all/archetypes/asset3d_simple.cpp b/docs/snippets/all/archetypes/asset3d_simple.cpp index 1689e792ef269..9d2c9893983bf 100644 --- a/docs/snippets/all/archetypes/asset3d_simple.cpp +++ b/docs/snippets/all/archetypes/asset3d_simple.cpp @@ -2,9 +2,7 @@ #include -#include #include -#include int main(int argc, char* argv[]) { if (argc < 2) { diff --git a/docs/snippets/all/archetypes/video_manual_frames.cpp b/docs/snippets/all/archetypes/video_manual_frames.cpp new file mode 100644 index 0000000000000..2c54634cca5de --- /dev/null +++ b/docs/snippets/all/archetypes/video_manual_frames.cpp @@ -0,0 +1,47 @@ +// Log a video asset using manually created frame references. +// TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. + +#include + +#include + +using namespace std::chrono_literals; + +int main(int argc, char* argv[]) { + if (argc < 2) { + // TODO(#7354): Only mp4 is supported for now. + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + + const auto path = argv[1]; + + const auto rec = rerun::RecordingStream("rerun_example_asset_video_manual_frames"); + rec.spawn().exit_on_failure(); + + // Log video asset which is referred to by frame references. + // Make sure it's available on the timeline used for the frame references. + rec.set_time_seconds("video_time", 0.0); + rec.log("video", rerun::AssetVideo::from_file(path).value_or_throw()); + + // Send frame references for every 0.1 seconds over a total of 10 seconds. + // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. + // TODO(#7368): Point to example using `send_video_frames`. + // + // Use `send_columns` to send all frame references in a single call. + std::vector times(10 * 10); + std::vector video_timestamps(10 * 10); + for (size_t i = 0; i < times.size(); i++) { + times[i] = 100ms * i; + video_timestamps[i] = rerun::components::VideoTimestamp(times[i]); + } + rec.send_columns( + "video", + rerun::TimeColumn::from_times("video_time", rerun::borrow(times)), + { + rerun::ComponentColumn::from_indicators(times.size()) + .value_or_throw(), + rerun::ComponentColumn::from_loggable(rerun::borrow(video_timestamps)).value_or_throw(), + } + ); +} diff --git a/docs/snippets/all/archetypes/video_manual_frames.py b/docs/snippets/all/archetypes/video_manual_frames.py new file mode 100644 index 0000000000000..e515c337ef527 --- /dev/null +++ b/docs/snippets/all/archetypes/video_manual_frames.py @@ -0,0 +1,33 @@ +""" +Log a video asset using manually created frame references. + +TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. +""" + +import sys + +import rerun as rr +import numpy as np + +if len(sys.argv) < 2: + # TODO(#7354): Only mp4 is supported for now. + print(f"Usage: {sys.argv[0]} ") + sys.exit(1) + +rr.init("rerun_example_asset_video_manual_frames", spawn=True) + +# Log video asset which is referred to by frame references. +rr.set_time_seconds("video_time", 0) # Make sure it's available on the timeline used for the frame references. +rr.log("video", rr.AssetVideo(path=sys.argv[1])) + +# Send frame references for every 0.1 seconds over a total of 10 seconds. +# Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. +# TODO(#7368): Point to example using `send_video_frames`. +# +# Use `send_columns` to send all frame references in a single call. +times = np.arange(0.0, 10.0, 0.1) +rr.send_columns( + "video", + times=[rr.TimeSecondsColumn("video_time", times)], + components=[rr.VideoFrameReference.indicator(), rr.components.VideoTimestamp.seconds(times)], +) diff --git a/docs/snippets/all/archetypes/video_manual_frames.rs b/docs/snippets/all/archetypes/video_manual_frames.rs new file mode 100644 index 0000000000000..24e7f25fe7993 --- /dev/null +++ b/docs/snippets/all/archetypes/video_manual_frames.rs @@ -0,0 +1,40 @@ +//! Log a video asset using manually created frame references. +//! TODO(#7298): ⚠️ Video is currently only supported in the Rerun web viewer. + +use rerun::{external::anyhow, TimeColumn}; + +fn main() -> anyhow::Result<()> { + let args = _args; + let Some(path) = args.get(1) else { + // TODO(#7354): Only mp4 is supported for now. + anyhow::bail!("Usage: {} ", args[0]); + }; + + let rec = + rerun::RecordingStreamBuilder::new("rerun_example_asset_video_manual_frames").spawn()?; + + // Log video asset which is referred to by frame references. + rec.set_time_seconds("video_time", 0.0); // Make sure it's available on the timeline used for the frame references. + rec.log("video", &rerun::AssetVideo::from_file_path(path)?)?; + + // Send frame references for every 0.1 seconds over a total of 10 seconds. + // Naturally, this will result in a choppy playback and only makes sense if the video is 10 seconds or longer. + // TODO(#7368): Point to example using `send_video_frames`. + // + // Use `send_columns` to send all frame references in a single call. + let times = (0..(10 * 10)).map(|t| t as f64 * 0.1).collect::>(); + let time_column = TimeColumn::new_seconds("video_time", times.iter().copied()); + let frame_reference_indicators = + ::Indicator::new_array(times.len()); + let video_timestamps = times + .into_iter() + .map(rerun::components::VideoTimestamp::from_seconds) + .collect::>(); + rec.send_columns( + "video", + [time_column], + [&frame_reference_indicators as _, &video_timestamps as _], + )?; + + Ok(()) +} diff --git a/docs/snippets/all/archetypes/video_simple.cpp b/docs/snippets/all/archetypes/video_simple.cpp deleted file mode 100644 index 82907b9ed9207..0000000000000 --- a/docs/snippets/all/archetypes/video_simple.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// Log a video file. - -#include - -#include -#include -#include - -int main(int argc, char* argv[]) { - if (argc < 2) { - std::cerr << "Usage: " << argv[0] << " " << std::endl; - return 1; - } - - const auto path = argv[1]; - - const auto rec = rerun::RecordingStream("rerun_example_asset_video"); - rec.spawn().exit_on_failure(); - - rec.log("world/video", rerun::AssetVideo::from_file(path).value_or_throw()); -} diff --git a/docs/snippets/all/archetypes/video_simple.py b/docs/snippets/all/archetypes/video_simple.py deleted file mode 100755 index a4ef1ac7fe283..0000000000000 --- a/docs/snippets/all/archetypes/video_simple.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Log a video file.""" - -import sys - -import rerun as rr - -if len(sys.argv) < 2: - print(f"Usage: {sys.argv[0]} ") - sys.exit(1) - -rr.init("rerun_example_asset_video", spawn=True) - -rr.log("world/video", rr.AssetVideo(path=sys.argv[1])) diff --git a/docs/snippets/all/archetypes/video_simple.rs b/docs/snippets/all/archetypes/video_simple.rs deleted file mode 100644 index 19abd3f954463..0000000000000 --- a/docs/snippets/all/archetypes/video_simple.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Log a video file. - -use rerun::external::anyhow; - -fn main() -> anyhow::Result<()> { - let args = std::env::args().collect::>(); - let Some(path) = args.get(1) else { - anyhow::bail!("Usage: {} ", args[0]); - }; - - let rec = rerun::RecordingStreamBuilder::new("rerun_example_asset_video").spawn()?; - - rec.log("world/video", &rerun::AssetVideo::from_file_path(path)?)?; - - Ok(()) -} diff --git a/docs/snippets/snippets.toml b/docs/snippets/snippets.toml index 535a82f4ca316..2be554f6a1bdf 100644 --- a/docs/snippets/snippets.toml +++ b/docs/snippets/snippets.toml @@ -188,4 +188,6 @@ quick_start = [ # These examples don't have exactly the same implementation. [extra_args] "archetypes/asset3d_simple" = ["$config_dir/../../tests/assets/cube.glb"] "archetypes/asset3d_out_of_tree" = ["$config_dir/../../tests/assets/cube.glb"] -"archetypes/video_simple" = ["$config_dir/../../tests/assets/empty.mp4"] +"archetypes/video_manual_frames" = [ + "$config_dir/../../tests/assets/video/Big_Buck_Bunny_1080_10s_av1.mp4", +] diff --git a/rerun_cpp/src/rerun/archetypes/asset3d.hpp b/rerun_cpp/src/rerun/archetypes/asset3d.hpp index 8ce9db4f95cf3..812cf71eacb94 100644 --- a/rerun_cpp/src/rerun/archetypes/asset3d.hpp +++ b/rerun_cpp/src/rerun/archetypes/asset3d.hpp @@ -33,9 +33,7 @@ namespace rerun::archetypes { /// ```cpp /// #include /// - /// #include /// #include - /// #include /// /// int main(int argc, char* argv[]) { /// if (argc <2) { diff --git a/rerun_cpp/src/rerun/components/video_timestamp.hpp b/rerun_cpp/src/rerun/components/video_timestamp.hpp index 7a09d14eba53b..446208a9f9812 100644 --- a/rerun_cpp/src/rerun/components/video_timestamp.hpp +++ b/rerun_cpp/src/rerun/components/video_timestamp.hpp @@ -6,6 +6,7 @@ #include "../datatypes/video_timestamp.hpp" #include "../result.hpp" +#include #include #include @@ -16,6 +17,39 @@ namespace rerun::components { struct VideoTimestamp { rerun::datatypes::VideoTimestamp timestamp; + public: // START of extensions from video_timestamp_ext.cpp: + /// Creates a new `VideoTimestamp` component. + /// \param video_time Timestamp value, type defined by `time_mode`. + /// \param time_mode How to interpret `video_time`. + VideoTimestamp(int64_t video_time, rerun::datatypes::VideoTimeMode time_mode) + : VideoTimestamp(rerun::datatypes::VideoTimestamp{video_time, time_mode}) {} + + /// Creates a new `VideoTimestamp` from time since video start. + /// \param time Time since video start. + template + VideoTimestamp(std::chrono::duration time) + : VideoTimestamp( + std::chrono::duration_cast(time).count(), + datatypes::VideoTimeMode::Nanoseconds + ) {} + + /// Creates a new [`VideoTimestamp`] from seconds since video start. + static VideoTimestamp from_seconds(double seconds) { + return VideoTimestamp(std::chrono::duration(seconds)); + } + + /// Creates a new [`VideoTimestamp`] from milliseconds since video start. + static VideoTimestamp from_milliseconds(double milliseconds) { + return VideoTimestamp(std::chrono::duration(milliseconds)); + } + + /// Creates a new [`VideoTimestamp`] from nanoseconds since video start. + static VideoTimestamp from_nanoseconds(int64_t nanoseconds) { + return VideoTimestamp(std::chrono::nanoseconds(nanoseconds)); + } + + // END of extensions from video_timestamp_ext.cpp, start of generated code: + public: VideoTimestamp() = default; diff --git a/rerun_cpp/src/rerun/components/video_timestamp_ext.cpp b/rerun_cpp/src/rerun/components/video_timestamp_ext.cpp new file mode 100644 index 0000000000000..d00165c231b03 --- /dev/null +++ b/rerun_cpp/src/rerun/components/video_timestamp_ext.cpp @@ -0,0 +1,50 @@ +#if 0 + +// +#include +// + +#include "../datatypes/video_time_mode.hpp" +#include "video_timestamp.hpp" + +namespace rerun { + namespace components { + + // + + /// Creates a new `VideoTimestamp` component. + /// \param video_time Timestamp value, type defined by `time_mode`. + /// \param time_mode How to interpret `video_time`. + VideoTimestamp(int64_t video_time, rerun::datatypes::VideoTimeMode time_mode) + : VideoTimestamp(rerun::datatypes::VideoTimestamp{video_time, time_mode}) {} + + /// Creates a new `VideoTimestamp` from time since video start. + /// \param time Time since video start. + template + VideoTimestamp(std::chrono::duration time) + : VideoTimestamp( + std::chrono::duration_cast(time).count(), + datatypes::VideoTimeMode::Nanoseconds + ) {} + + /// Creates a new [`VideoTimestamp`] from seconds since video start. + static VideoTimestamp from_seconds(double seconds) { + return VideoTimestamp(std::chrono::duration(seconds)); + } + + /// Creates a new [`VideoTimestamp`] from milliseconds since video start. + static VideoTimestamp from_milliseconds(double milliseconds) { + return VideoTimestamp(std::chrono::duration(milliseconds)); + } + + /// Creates a new [`VideoTimestamp`] from nanoseconds since video start. + static VideoTimestamp from_nanoseconds(int64_t nanoseconds) { + return VideoTimestamp(std::chrono::nanoseconds(nanoseconds)); + } + + // + + } // namespace components +} // namespace rerun + +#endif diff --git a/rerun_py/rerun_sdk/rerun/__init__.py b/rerun_py/rerun_sdk/rerun/__init__.py index c40e195a7faeb..05856117a5422 100644 --- a/rerun_py/rerun_sdk/rerun/__init__.py +++ b/rerun_py/rerun_sdk/rerun/__init__.py @@ -79,6 +79,7 @@ TextDocument as TextDocument, TextLog as TextLog, Transform3D as Transform3D, + VideoFrameReference as VideoFrameReference, ViewCoordinates as ViewCoordinates, ) from .archetypes.boxes2d_ext import ( diff --git a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py index 433fe013f8ccd..23b11eae4ddca 100644 --- a/rerun_py/rerun_sdk/rerun/components/video_timestamp.py +++ b/rerun_py/rerun_sdk/rerun/components/video_timestamp.py @@ -10,11 +10,12 @@ ComponentBatchMixin, ComponentMixin, ) +from .video_timestamp_ext import VideoTimestampExt __all__ = ["VideoTimestamp", "VideoTimestampBatch", "VideoTimestampType"] -class VideoTimestamp(datatypes.VideoTimestamp, ComponentMixin): +class VideoTimestamp(VideoTimestampExt, datatypes.VideoTimestamp, ComponentMixin): """ **Component**: Timestamp inside a [`archetypes.AssetVideo`][rerun.archetypes.AssetVideo]. diff --git a/rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py b/rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py new file mode 100644 index 0000000000000..348e8730e5d64 --- /dev/null +++ b/rerun_py/rerun_sdk/rerun/components/video_timestamp_ext.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import numpy as np +import numpy.typing as npt + +from .. import components, datatypes + + +class VideoTimestampExt: + """Extension for [VideoTimestamp][rerun.components.VideoTimestamp].""" + + # Implementation note: + # We could add an init method that deals with seconds/milliseconds/nanoseconds etc. + # However, this would require _a lot_ of slow parameter validation on a per timestamp basis. + # When in actuallity, this data practifally always comes in homogenous batches. + + @staticmethod + def seconds( + seconds: npt.ArrayLike, + ) -> components.VideoTimestampBatch: + """ + Create a video timestamp batch from seconds since video start. + + Parameters + ---------- + seconds: + Timestamp values in seconds since video start. + + """ + return components.VideoTimestamp.nanoseconds(np.array(seconds) * 1e9) + + @staticmethod + def milliseconds( + milliseconds: npt.ArrayLike, + ) -> components.VideoTimestampBatch: + """ + Create a video timestamp batch from milliseconds since video start. + + Parameters + ---------- + milliseconds: + Timestamp values in milliseconds since video start. + + """ + return components.VideoTimestamp.nanoseconds(np.array(milliseconds) * 1e6) + + @staticmethod + def nanoseconds( + nanoseconds: npt.ArrayLike, + ) -> components.VideoTimestampBatch: + """ + Create a video timestamp batch from nanoseconds since video start. + + Parameters + ---------- + nanoseconds: + Timestamp values in milliseconds since video start. + + """ + nanoseconds = np.asarray(nanoseconds, dtype=np.int64) + + return components.VideoTimestampBatch([ + components.VideoTimestamp(video_time=ns, time_mode=datatypes.VideoTimeMode.Nanoseconds) + for ns in nanoseconds + ])