Update the dataframe view to use re_dataframe and egui_table (#7380)

### What - Closes #7279 Major update to the dataframe view - display the data return by the new `re_dataframe` crate - the PoV entity/component is now actually used - entities are now always columns - see #7379 - use [`egui_table`](https://github.com/rerun-io/egui_table) for the table - hierarchical header - sticky columns - and much more... TODO: - [x] fix after merging #7383 <img width="2670" alt="image" src="https://github.com/user-attachments/assets/cf09b69b-3c82-4ba9-9425-bf60622efae4"> ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7380?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7380?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7380) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --------- Co-authored-by: Emil Ernerfeldt <[email protected]> Co-authored-by: Clement Rey <[email protected]>
rerun-io · Sep 10, 2024 · e9b02e3 · e9b02e3
1 parent ef26bf8
commit e9b02e3
Show file tree

Hide file tree

Showing 25 changed files with 864 additions and 705 deletions.
diff --git a/Cargo.lock b/Cargo.lock
@@ -1777,7 +1777,7 @@ checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30"
 [[package]]
 name = "ecolor"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "bytemuck",
  "emath",
@@ -1787,7 +1787,7 @@ dependencies = [
 [[package]]
 name = "eframe"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "ahash",
  "bytemuck",
@@ -1824,7 +1824,7 @@ dependencies = [
 [[package]]
 name = "egui"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "accesskit",
  "ahash",
@@ -1841,7 +1841,7 @@ dependencies = [
 [[package]]
 name = "egui-wgpu"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "ahash",
  "bytemuck",
@@ -1860,7 +1860,7 @@ dependencies = [
 [[package]]
 name = "egui-winit"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "accesskit_winit",
  "ahash",
@@ -1902,7 +1902,7 @@ dependencies = [
 [[package]]
 name = "egui_extras"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "ahash",
  "egui",
@@ -1918,7 +1918,7 @@ dependencies = [
 [[package]]
 name = "egui_glow"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "ahash",
  "bytemuck",
@@ -1944,6 +1944,16 @@ dependencies = [
  "emath",
 ]
 
+[[package]]
+name = "egui_table"
+version = "0.28.1"
+source = "git+https://github.com/rerun-io/egui_table.git?rev=0f594701d528c4a9553521cb941de1886549dc70#0f594701d528c4a9553521cb941de1886549dc70"
+dependencies = [
+ "egui",
+ "serde",
+ "vec1",
+]
+
 [[package]]
 name = "egui_tiles"
 version = "0.9.1"
@@ -1981,7 +1991,7 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
 [[package]]
 name = "emath"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "bytemuck",
  "serde",
@@ -2082,7 +2092,7 @@ dependencies = [
 [[package]]
 name = "epaint"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 dependencies = [
  "ab_glyph",
  "ahash",
@@ -2101,7 +2111,7 @@ dependencies = [
 [[package]]
 name = "epaint_default_fonts"
 version = "0.28.1"
-source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906"
+source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66"
 
 [[package]]
 name = "equivalent"
@@ -5047,6 +5057,7 @@ dependencies = [
  "re_build_info",
  "re_chunk",
  "re_chunk_store",
+ "re_dataframe",
  "re_format",
  "re_int_histogram",
  "re_log",
@@ -5448,11 +5459,14 @@ dependencies = [
 name = "re_space_view_dataframe"
 version = "0.19.0-alpha.1+dev"
 dependencies = [
+ "anyhow",
  "egui",
  "egui_extras",
+ "egui_table",
  "itertools 0.13.0",
  "re_chunk_store",
  "re_data_ui",
+ "re_dataframe",
  "re_entity_db",
  "re_log",
  "re_log_types",
@@ -5464,6 +5478,7 @@ dependencies = [
  "re_ui",
  "re_viewer_context",
  "re_viewport_blueprint",
+ "thiserror",
 ]
 
 [[package]]
@@ -7488,9 +7503,9 @@ dependencies = [
 
 [[package]]
 name = "vec1"
-version = "1.10.1"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bda7c41ca331fe9a1c278a9e7ee055f4be7f5eb1c2b72f079b4ff8b5fce9d5c"
+checksum = "eab68b56840f69efb0fefbe3ab6661499217ffdc58e2eef7c3f6f69835386322"
 dependencies = [
  "smallvec",
 ]

diff --git a/Cargo.toml b/Cargo.toml
@@ -44,6 +44,7 @@ re_chunk = { path = "crates/store/re_chunk", version = "=0.19.0-alpha.1", defaul
 re_chunk_store = { path = "crates/store/re_chunk_store", version = "=0.19.0-alpha.1", default-features = false }
 re_data_loader = { path = "crates/store/re_data_loader", version = "=0.19.0-alpha.1", default-features = false }
 re_data_source = { path = "crates/store/re_data_source", version = "=0.19.0-alpha.1", default-features = false }
+re_dataframe = { path = "crates/store/re_dataframe", version = "=0.19.0-alpha.1", default-features = false }
 re_entity_db = { path = "crates/store/re_entity_db", version = "=0.19.0-alpha.1", default-features = false }
 re_format_arrow = { path = "crates/store/re_format_arrow", version = "=0.19.0-alpha.1", default-features = false }
 re_log_encoding = { path = "crates/store/re_log_encoding", version = "=0.19.0-alpha.1", default-features = false }
@@ -513,12 +514,12 @@ missing_errors_doc = "allow"
 # As a last resport, patch with a commit to our own repository.
 # ALWAYS document what PR the commit hash is part of, or when it was merged into the upstream trunk.
 
-ecolor = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" }      # egui master 2024-09-03
-eframe = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" }      # egui master 2024-09-03
-egui = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" }        # egui master 2024-09-03
-egui_extras = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" } # egui master 2024-09-03
-egui-wgpu = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" }   # egui master 2024-09-03
-emath = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" }       # egui master 2024-09-03
+ecolor = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }      # egui master 2024-09-06
+eframe = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }      # egui master 2024-09-06
+egui = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }        # egui master 2024-09-06
+egui_extras = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06
+egui-wgpu = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }   # egui master 2024-09-06
+emath = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" }       # egui master 2024-09-06
 
 # Useful while developing:
 # ecolor = { path = "../../egui/crates/ecolor" }

diff --git a/crates/store/re_chunk/src/transport.rs b/crates/store/re_chunk/src/transport.rs
@@ -273,6 +273,15 @@ impl TransportChunk {
             })
     }
 
+    #[inline]
+    pub fn all_columns(&self) -> impl Iterator<Item = (&ArrowField, &Box<dyn ArrowArray>)> + '_ {
+        self.schema
+            .fields
+            .iter()
+            .enumerate()
+            .filter_map(|(i, field)| self.data.columns().get(i).map(|column| (field, column)))
+    }
+
     /// Iterates all control columns present in this chunk.
     #[inline]
     pub fn controls(&self) -> impl Iterator<Item = (&ArrowField, &Box<dyn ArrowArray>)> {

diff --git a/crates/store/re_chunk_store/src/dataframe.rs b/crates/store/re_chunk_store/src/dataframe.rs
@@ -65,14 +65,23 @@ impl ColumnDescriptor {
             Self::Component(descr) => descr.to_arrow_field(),
         }
     }
+
+    #[inline]
+    pub fn short_name(&self) -> String {
+        match self {
+            Self::Control(descr) => descr.component_name.short_name().to_owned(),
+            Self::Time(descr) => descr.timeline.name().to_string(),
+            Self::Component(descr) => descr.component_name.short_name().to_owned(),
+        }
+    }
 }
 
 /// Describes a column used to control Rerun's behavior, such as `RowId`.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct ControlColumnDescriptor {
     /// Semantic name associated with this data.
     ///
-    /// Example: `rerun.controls.RowId`.
+    /// Example: `RowId::name()`.
     pub component_name: ComponentName,
 
     /// The Arrow datatype of the column.

diff --git a/crates/store/re_chunk_store/src/lib.rs b/crates/store/re_chunk_store/src/lib.rs
@@ -45,6 +45,8 @@ pub use re_log_encoding::decoder::VersionPolicy;
 pub use re_log_types::{ResolvedTimeRange, TimeInt, TimeType, Timeline};
 
 pub mod external {
+    pub use arrow2;
+
     pub use re_chunk;
     pub use re_log_encoding;
 }

diff --git a/crates/store/re_dataframe/src/latest_at.rs b/crates/store/re_dataframe/src/latest_at.rs
@@ -56,6 +56,11 @@ impl<'a> LatestAtQueryHandle<'a> {
 }
 
 impl LatestAtQueryHandle<'_> {
+    /// The query expression used to instantiate this handle.
+    pub fn query(&self) -> &LatestAtQueryExpression {
+        &self.query
+    }
+
     /// All results returned by this handle will strictly follow this schema.
     ///
     /// Columns that do not yield any data will still be present in the results, filled with null values.

diff --git a/crates/store/re_dataframe/src/range.rs b/crates/store/re_dataframe/src/range.rs
@@ -133,6 +133,11 @@ impl RangeQueryHandle<'_> {
         })
     }
 
+    /// The query used to instantiate this handle.
+    pub fn query(&self) -> &RangeQueryExpression {
+        &self.query
+    }
+
     /// All results returned by this handle will strictly follow this schema.
     ///
     /// Columns that do not yield any data will still be present in the results, filled with null values.

diff --git a/crates/store/re_entity_db/Cargo.toml b/crates/store/re_entity_db/Cargo.toml
@@ -30,6 +30,7 @@ serde = ["dep:serde", "re_log_types/serde"]
 re_build_info.workspace = true
 re_chunk = { workspace = true, features = ["serde"] }
 re_chunk_store.workspace = true
+re_dataframe.workspace = true
 re_format.workspace = true
 re_int_histogram.workspace = true
 re_log.workspace = true

diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs
@@ -121,6 +121,13 @@ impl EntityDb {
         &self.query_caches
     }
 
+    pub fn query_engine(&self) -> re_dataframe::QueryEngine<'_> {
+        re_dataframe::QueryEngine {
+            store: self.store(),
+            cache: self.query_caches(),
+        }
+    }
+
     /// Queries for the given `component_names` using latest-at semantics.
     ///
     /// See [`re_query::LatestAtResults`] for more information about how to handle the results.

diff --git a/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs b/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs
@@ -1,7 +1,7 @@
 use itertools::Itertools;
 
-use re_chunk_store::external::re_chunk::external::arrow2;
-use re_chunk_store::external::re_chunk::external::arrow2::array::Utf8Array;
+use re_chunk_store::external::arrow2;
+use re_chunk_store::external::arrow2::array::Utf8Array;
 use re_types::SizeBytes as _;
 use re_ui::UiExt;
 

diff --git a/crates/viewer/re_chunk_store_ui/src/chunk_list_mode.rs b/crates/viewer/re_chunk_store_ui/src/chunk_list_mode.rs
@@ -70,7 +70,7 @@ impl ChunkListMode {
                                 ..
                             }
                         ),
-                        "Latest at",
+                        "Latest-at",
                     )
                     .on_hover_text("Display chunks relevant to the provided latest-at query")
                     .clicked()

diff --git a/crates/viewer/re_chunk_store_ui/src/chunk_store_ui.rs b/crates/viewer/re_chunk_store_ui/src/chunk_store_ui.rs
@@ -334,8 +334,6 @@ impl DatastoreUi {
                     )
                     .resizable(true)
                     .vscroll(true)
-                    //TODO(ab): remove when https://github.com/emilk/egui/pull/4817 is merged/released
-                    .max_scroll_height(f32::INFINITY)
                     .auto_shrink([false, false])
                     .striped(true);
 

diff --git a/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs b/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs
@@ -171,8 +171,6 @@ impl ChunkUi {
                     )
                     .resizable(true)
                     .vscroll(true)
-                    //TODO(ab): remove when https://github.com/emilk/egui/pull/4817 is merged/released
-                    .max_scroll_height(f32::INFINITY)
                     .auto_shrink([false, false])
                     .striped(true);
 

diff --git a/crates/viewer/re_space_view_dataframe/Cargo.toml b/crates/viewer/re_space_view_dataframe/Cargo.toml
@@ -21,6 +21,7 @@ all-features = true
 [dependencies]
 re_chunk_store.workspace = true
 re_data_ui.workspace = true
+re_dataframe.workspace = true
 re_entity_db.workspace = true
 re_log.workspace = true
 re_log_types.workspace = true
@@ -33,6 +34,9 @@ re_ui.workspace = true
 re_viewer_context.workspace = true
 re_viewport_blueprint.workspace = true
 
+anyhow.workspace = true
 egui_extras.workspace = true
+egui_table = { git = "https://github.com/rerun-io/egui_table.git", rev = "0f594701d528c4a9553521cb941de1886549dc70" } # main as of 2024-09-09
 egui.workspace = true
 itertools.workspace = true
+thiserror.workspace = true