Merge branch 'main' into antoine/dataframe-instance-expansion

rerun-io · Sep 17, 2024 · 1f25fb6 · 1f25fb6
2 parents 89313e5 + 9fe16ba
commit 1f25fb6
Show file tree

Hide file tree

Showing 146 changed files with 2,189 additions and 564 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -4946,7 +4946,6 @@ dependencies = [
  "re_smart_channel",
  "re_tracing",
  "re_types",
- "re_video",
  "thiserror",
  "walkdir",
 ]
@@ -5799,8 +5798,10 @@ dependencies = [
 name = "re_video"
 version = "0.19.0-alpha.1+dev"
 dependencies = [
+ "itertools 0.13.0",
  "mp4",
  "ordered-float",
+ "thiserror",
 ]
 
 [[package]]
@@ -6156,6 +6157,7 @@ dependencies = [
  "re_arrow2",
  "re_log",
  "re_sdk",
+ "re_video",
 ]
 
 [[package]]
@@ -6180,6 +6182,7 @@ dependencies = [
  "re_log_types",
  "re_memory",
  "re_sdk",
+ "re_video",
  "re_web_viewer_server",
  "re_ws_comms",
  "uuid",

diff --git a/crates/build/re_types_builder/src/codegen/cpp/mod.rs b/crates/build/re_types_builder/src/codegen/cpp/mod.rs
@@ -1505,7 +1505,13 @@ fn to_arrow_method(
         (
             true,
             quote! {
-                return Loggable<#forwarded_type>::to_arrow(&instances->#field_name, num_instances);
+                if (num_instances == 0) {
+                    return Loggable<#forwarded_type>::to_arrow(nullptr, 0);
+                } else if (instances == nullptr) {
+                    return rerun::Error(ErrorCode::UnexpectedNullArgument, "Passed array instances is null when num_elements > 0.");
+                } else {
+                    return Loggable<#forwarded_type>::to_arrow(&instances->#field_name, num_instances);
+                }
             },
         )
     } else {

diff --git a/crates/store/README.md b/crates/store/README.md
@@ -1 +1 @@
-Creates related to storing, indexing, trasmitting, and handling data.
+Creates related to storing, indexing, transmitting, and handling data.
diff --git a/crates/store/re_data_loader/Cargo.toml b/crates/store/re_data_loader/Cargo.toml
@@ -31,8 +31,7 @@ re_log_types.workspace = true
 re_log.workspace = true
 re_smart_channel.workspace = true
 re_tracing.workspace = true
-re_types = { workspace = true, features = ["image"] }
-re_video.workspace = true
+re_types = { workspace = true, features = ["image", "video"] }
 
 ahash.workspace = true
 anyhow.workspace = true

diff --git a/crates/store/re_data_loader/src/loader_archetype.rs b/crates/store/re_data_loader/src/loader_archetype.rs
@@ -1,12 +1,11 @@
 use re_chunk::{Chunk, RowId};
 use re_log_types::{EntityPath, TimeInt, TimePoint};
-use re_types::archetypes::VideoFrameReference;
+use re_types::archetypes::{AssetVideo, VideoFrameReference};
+use re_types::components::VideoTimestamp;
 use re_types::Archetype;
 use re_types::{components::MediaType, ComponentBatch};
 
-use arrow2::array::{
-    ListArray as ArrowListArray, NullArray as ArrowNullArray, PrimitiveArray as ArrowPrimitiveArray,
-};
+use arrow2::array::PrimitiveArray as ArrowPrimitiveArray;
 use arrow2::Either;
 
 use crate::{DataLoader, DataLoaderError, LoadedData};
@@ -220,100 +219,60 @@ fn load_video(
     let video_timeline = re_log_types::Timeline::new_temporal("video");
     timepoint.insert(video_timeline, re_log_types::TimeInt::new_temporal(0));
 
-    let media_type = MediaType::guess_from_path(filepath);
-
-    // TODO(andreas): Video frame reference generation should be available as a utility from the SDK.
-
-    let video = if media_type.as_ref().map(|v| v.as_str()) == Some("video/mp4") {
-        match re_video::load_mp4(&contents) {
-            Ok(video) => Some(video),
-            Err(err) => {
-                re_log::warn!("Failed to load video asset {filepath:?}: {err}");
-                None
-            }
+    let video_asset = AssetVideo::new(contents);
+
+    let video_frame_reference_chunk = match video_asset.read_frame_timestamps_ns() {
+        Ok(frame_timestamps_ns) => {
+            // Time column.
+            let is_sorted = Some(true);
+            let time_column_times = ArrowPrimitiveArray::from_slice(&frame_timestamps_ns);
+            let time_column =
+                re_chunk::TimeColumn::new(is_sorted, video_timeline, time_column_times);
+
+            // VideoTimestamp component column.
+            let video_timestamps = frame_timestamps_ns
+                .into_iter()
+                .map(VideoTimestamp::from_nanoseconds)
+                .collect::<Vec<_>>();
+            let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch;
+            let video_timestamp_list_array = video_timestamp_batch
+                .to_arrow_list_array()
+                .map_err(re_chunk::ChunkError::from)?;
+
+            // Indicator column.
+            let video_frame_reference_indicators =
+                <VideoFrameReference as Archetype>::Indicator::new_array(video_timestamps.len());
+            let video_frame_reference_indicators_list_array = video_frame_reference_indicators
+                .to_arrow_list_array()
+                .map_err(re_chunk::ChunkError::from)?;
+
+            Some(Chunk::from_auto_row_ids(
+                re_chunk::ChunkId::new(),
+                entity_path.clone(),
+                std::iter::once((video_timeline, time_column)).collect(),
+                [
+                    (
+                        VideoFrameReference::indicator().name(),
+                        video_frame_reference_indicators_list_array,
+                    ),
+                    (video_timestamp_batch.name(), video_timestamp_list_array),
+                ]
+                .into_iter()
+                .collect(),
+            )?)
         }
-    } else {
-        re_log::warn!("Video asset {filepath:?} has an unsupported container format.");
-        None
-    };
 
-    // Log video frame references on the `video` timeline.
-    let video_frame_reference_chunk = if let Some(video) = video {
-        let first_timestamp = video
-            .segments
-            .first()
-            .map_or(0, |segment| segment.timestamp.as_nanoseconds());
-
-        // Time column.
-        let is_sorted = Some(true);
-        let time_column_times =
-            ArrowPrimitiveArray::<i64>::from_values(video.segments.iter().flat_map(|segment| {
-                segment
-                    .samples
-                    .iter()
-                    .map(|s| s.timestamp.as_nanoseconds() - first_timestamp)
-            }));
-
-        let time_column = re_chunk::TimeColumn::new(is_sorted, video_timeline, time_column_times);
-
-        // VideoTimestamp component column.
-        let video_timestamps = video
-            .segments
-            .iter()
-            .flat_map(|segment| {
-                segment.samples.iter().map(|s| {
-                    // TODO(andreas): Use sample indices instead of timestamps once possible.
-                    re_types::components::VideoTimestamp::from_nanoseconds(
-                        s.timestamp.as_nanoseconds(),
-                    )
-                })
-            })
-            .collect::<Vec<_>>();
-        let video_timestamp_batch = &video_timestamps as &dyn ComponentBatch;
-        let video_timestamp_list_array = video_timestamp_batch
-            .to_arrow_list_array()
-            .map_err(re_chunk::ChunkError::from)?;
-
-        // Indicator column.
-        let video_frame_reference_indicator_datatype = arrow2::datatypes::DataType::Null;
-        let video_frame_reference_indicator_list_array = ArrowListArray::<i32>::try_new(
-            ArrowListArray::<i32>::default_datatype(
-                video_frame_reference_indicator_datatype.clone(),
-            ),
-            video_timestamp_list_array.offsets().clone(),
-            Box::new(ArrowNullArray::new(
-                video_frame_reference_indicator_datatype,
-                video_timestamps.len(),
-            )),
-            None,
-        )
-        .map_err(re_chunk::ChunkError::from)?;
-
-        Some(Chunk::from_auto_row_ids(
-            re_chunk::ChunkId::new(),
-            entity_path.clone(),
-            std::iter::once((video_timeline, time_column)).collect(),
-            [
-                (
-                    VideoFrameReference::indicator().name(),
-                    video_frame_reference_indicator_list_array,
-                ),
-                (video_timestamp_batch.name(), video_timestamp_list_array),
-            ]
-            .into_iter()
-            .collect(),
-        )?)
-    } else {
-        None
+        Err(err) => {
+            re_log::warn_once!(
+                "Failed to read frame timestamps from video asset {filepath:?}: {err}"
+            );
+            None
+        }
     };
 
     // Put video asset into its own chunk since it can be fairly large.
     let video_asset_chunk = Chunk::builder(entity_path.clone())
-        .with_archetype(
-            RowId::new(),
-            timepoint.clone(),
-            &re_types::archetypes::AssetVideo::from_file_contents(contents, media_type.clone()),
-        )
+        .with_archetype(RowId::new(), timepoint.clone(), &video_asset)
         .with_component_batch(RowId::new(), timepoint.clone(), &ExperimentalFeature)
         .build()?;
 

diff --git a/crates/store/re_types/Cargo.toml b/crates/store/re_types/Cargo.toml
@@ -25,7 +25,7 @@ features = ["all"]
 default = ["ecolor"]
 
 ## All features except `testing`.
-all = ["ecolor", "egui_plot", "glam", "image", "mint", "serde"]
+all = ["ecolor", "egui_plot", "glam", "image", "mint", "serde", "video"]
 
 ## Enable color conversions.
 ecolor = ["dep:ecolor"]
@@ -39,7 +39,7 @@ glam = ["dep:glam"]
 ## Integration with the [`image`](https://crates.io/crates/image/) crate, plus JPEG support.
 image = ["dep:ecolor", "dep:image"]
 
-## Conversion to/from our video format
+## Inspecting video data.
 video = ["dep:re_video"]
 
 ## Enable (de)serialization using serde.

diff --git a/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs b/crates/store/re_types/definitions/rerun/archetypes/asset_video.fbs
@@ -8,9 +8,11 @@ namespace rerun.archetypes;
 ///
 /// In order to display a video, you need to log a [archetypes.VideoFrameReference] for each frame.
 ///
-/// \example archetypes/video_manual_frames title="Video with explicit frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
-// TODO(#7368): Example and reference to `send_video_frames` API.
+/// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
+/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
+// TODO(#7420): update screenshot for manual frames example
 table AssetVideo (
+  "attr.docs.unreleased",
   "attr.rerun.experimental"
 ) {
   // --- Required ---

diff --git a/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs b/crates/store/re_types/definitions/rerun/archetypes/video_frame_reference.fbs
@@ -5,9 +5,11 @@ namespace rerun.archetypes;
 /// Used to display individual video frames from a [archetypes.AssetVideo].
 /// To show an entire video, a fideo frame reference for each frame of the video should be logged.
 ///
-/// \example archetypes/video_manual_frames title="Video with explicit frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
-// TODO(#7368): Example and reference to `send_video_frames` API.
+/// \example archetypes/video_auto_frames title="Video with automatically determined frames" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
+/// \example archetypes/video_manual_frames title="Demonstrates manual use of video frame references" image="https://static.rerun.io/video_manual_frames/320a44e1e06b8b3a3161ecbbeae3e04d1ccb9589/1200w.png"
+// TODO(#7420): update screenshot for manual frames example
 table VideoFrameReference (
+    "attr.docs.unreleased",
     "attr.rerun.experimental"
 ){
     // --- Required ---

diff --git a/crates/store/re_types/definitions/rerun/components/entity_path.fbs b/crates/store/re_types/definitions/rerun/components/entity_path.fbs
@@ -2,6 +2,7 @@ namespace rerun.components;
 
 /// A path to an entity, usually to reference some data that is part of the target entity.
 table EntityPath (
+  "attr.docs.unreleased",
   "attr.arrow.transparent",
   "attr.python.aliases": "str",
   "attr.python.array_aliases": "str, Sequence[str]",

diff --git a/crates/store/re_types/definitions/rerun/components/video_timestamp.fbs b/crates/store/re_types/definitions/rerun/components/video_timestamp.fbs
@@ -3,6 +3,7 @@ namespace rerun.components;
 
 /// Timestamp inside a [archetypes.AssetVideo].
 struct VideoTimestamp (
+  "attr.docs.unreleased",
   "attr.rust.derive": "Copy, PartialEq, Eq, Default",
   "attr.rust.repr": "transparent",
   "attr.rerun.experimental"

diff --git a/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs b/crates/store/re_types/definitions/rerun/datatypes/video_timestamp.fbs
@@ -1,7 +1,9 @@
 namespace rerun.datatypes;
 
 /// Specifies how to interpret the `video_time` field of a [datatypes.VideoTimestamp].
-enum VideoTimeMode: ubyte{
+enum VideoTimeMode: ubyte (
+  "attr.docs.unreleased"
+) {
     /// Invalid value. Won't show up in generated types.
     Invalid = 0,
 
@@ -13,6 +15,7 @@ enum VideoTimeMode: ubyte{
 
 /// Timestamp inside a [archetypes.AssetVideo].
 struct VideoTimestamp (
+  "attr.docs.unreleased",
   "attr.rust.derive": "Copy, PartialEq, Eq",
   "attr.rerun.experimental"
 ) {
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		Creates related to storing, indexing, trasmitting, and handling data.
		Creates related to storing, indexing, transmitting, and handling data.