From e9b02e304d8e90ac2f71011c46f2874ec4f63939 Mon Sep 17 00:00:00 2001 From: Antoine Beyeler <49431240+abey79@users.noreply.github.com> Date: Tue, 10 Sep 2024 14:29:48 +0200 Subject: [PATCH] Update the dataframe view to use `re_dataframe` and `egui_table` (#7380) ### What - Closes #7279 Major update to the dataframe view - display the data return by the new `re_dataframe` crate - the PoV entity/component is now actually used - entities are now always columns - see #7379 - use [`egui_table`](https://github.com/rerun-io/egui_table) for the table - hierarchical header - sticky columns - and much more... TODO: - [x] fix after merging #7383 image ### Checklist * [x] I have read and agree to [Contributor Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and the [Code of Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md) * [x] I've included a screenshot or gif (if applicable) * [x] I have tested the web demo (if applicable): * Using examples from latest `main` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7380?manifest_url=https://app.rerun.io/version/main/examples_manifest.json) * Using full set of examples from `nightly` build: [rerun.io/viewer](https://rerun.io/viewer/pr/7380?manifest_url=https://app.rerun.io/version/nightly/examples_manifest.json) * [x] The PR title and labels are set such as to maximize their usefulness for the next release's CHANGELOG * [x] If applicable, add a new check to the [release checklist](https://github.com/rerun-io/rerun/blob/main/tests/python/release_checklist)! * [x] If have noted any breaking changes to the log API in `CHANGELOG.md` and the migration guide - [PR Build Summary](https://build.rerun.io/pr/7380) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html) To run all checks from `main`, comment on the PR with `@rerun-bot full-check`. --------- Co-authored-by: Emil Ernerfeldt Co-authored-by: Clement Rey --- Cargo.lock | 39 +- Cargo.toml | 13 +- crates/store/re_chunk/src/transport.rs | 9 + crates/store/re_chunk_store/src/dataframe.rs | 11 +- crates/store/re_chunk_store/src/lib.rs | 2 + crates/store/re_dataframe/src/latest_at.rs | 5 + crates/store/re_dataframe/src/range.rs | 5 + crates/store/re_entity_db/Cargo.toml | 1 + crates/store/re_entity_db/src/entity_db.rs | 7 + .../viewer/re_chunk_store_ui/src/arrow_ui.rs | 4 +- .../re_chunk_store_ui/src/chunk_list_mode.rs | 2 +- .../re_chunk_store_ui/src/chunk_store_ui.rs | 2 - .../viewer/re_chunk_store_ui/src/chunk_ui.rs | 2 - .../viewer/re_space_view_dataframe/Cargo.toml | 4 + .../src/dataframe_ui.rs | 350 ++++++++++++++++++ .../src/display_record_batch.rs | 344 +++++++++++++++++ .../src/latest_at_table.rs | 217 ----------- .../viewer/re_space_view_dataframe/src/lib.rs | 6 +- .../src/space_view_class.rs | 96 +++-- .../re_space_view_dataframe/src/table_ui.rs | 67 ---- .../src/time_range_table.rs | 288 -------------- .../re_space_view_dataframe/src/utils.rs | 58 --- crates/viewer/re_viewer/src/app.rs | 7 +- crates/viewer/re_viewer/src/web.rs | 1 + .../src/component_ui_registry.rs | 29 +- 25 files changed, 864 insertions(+), 705 deletions(-) create mode 100644 crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs create mode 100644 crates/viewer/re_space_view_dataframe/src/display_record_batch.rs delete mode 100644 crates/viewer/re_space_view_dataframe/src/latest_at_table.rs delete mode 100644 crates/viewer/re_space_view_dataframe/src/table_ui.rs delete mode 100644 crates/viewer/re_space_view_dataframe/src/time_range_table.rs delete mode 100644 crates/viewer/re_space_view_dataframe/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index d30473d9371a..ccf83cfd8887 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1777,7 +1777,7 @@ checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" [[package]] name = "ecolor" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "bytemuck", "emath", @@ -1787,7 +1787,7 @@ dependencies = [ [[package]] name = "eframe" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "ahash", "bytemuck", @@ -1824,7 +1824,7 @@ dependencies = [ [[package]] name = "egui" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "accesskit", "ahash", @@ -1841,7 +1841,7 @@ dependencies = [ [[package]] name = "egui-wgpu" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "ahash", "bytemuck", @@ -1860,7 +1860,7 @@ dependencies = [ [[package]] name = "egui-winit" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "accesskit_winit", "ahash", @@ -1902,7 +1902,7 @@ dependencies = [ [[package]] name = "egui_extras" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "ahash", "egui", @@ -1918,7 +1918,7 @@ dependencies = [ [[package]] name = "egui_glow" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "ahash", "bytemuck", @@ -1944,6 +1944,16 @@ dependencies = [ "emath", ] +[[package]] +name = "egui_table" +version = "0.28.1" +source = "git+https://github.com/rerun-io/egui_table.git?rev=0f594701d528c4a9553521cb941de1886549dc70#0f594701d528c4a9553521cb941de1886549dc70" +dependencies = [ + "egui", + "serde", + "vec1", +] + [[package]] name = "egui_tiles" version = "0.9.1" @@ -1981,7 +1991,7 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "emath" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "bytemuck", "serde", @@ -2082,7 +2092,7 @@ dependencies = [ [[package]] name = "epaint" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" dependencies = [ "ab_glyph", "ahash", @@ -2101,7 +2111,7 @@ dependencies = [ [[package]] name = "epaint_default_fonts" version = "0.28.1" -source = "git+https://github.com/emilk/egui.git?rev=454abf705b87aba70cef582d6ce80f74aa398906#454abf705b87aba70cef582d6ce80f74aa398906" +source = "git+https://github.com/emilk/egui.git?rev=6b7f4312373a301a4cdf7d99a0d546acd34bcd66#6b7f4312373a301a4cdf7d99a0d546acd34bcd66" [[package]] name = "equivalent" @@ -5047,6 +5057,7 @@ dependencies = [ "re_build_info", "re_chunk", "re_chunk_store", + "re_dataframe", "re_format", "re_int_histogram", "re_log", @@ -5448,11 +5459,14 @@ dependencies = [ name = "re_space_view_dataframe" version = "0.19.0-alpha.1+dev" dependencies = [ + "anyhow", "egui", "egui_extras", + "egui_table", "itertools 0.13.0", "re_chunk_store", "re_data_ui", + "re_dataframe", "re_entity_db", "re_log", "re_log_types", @@ -5464,6 +5478,7 @@ dependencies = [ "re_ui", "re_viewer_context", "re_viewport_blueprint", + "thiserror", ] [[package]] @@ -7488,9 +7503,9 @@ dependencies = [ [[package]] name = "vec1" -version = "1.10.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bda7c41ca331fe9a1c278a9e7ee055f4be7f5eb1c2b72f079b4ff8b5fce9d5c" +checksum = "eab68b56840f69efb0fefbe3ab6661499217ffdc58e2eef7c3f6f69835386322" dependencies = [ "smallvec", ] diff --git a/Cargo.toml b/Cargo.toml index 4c3c712309de..8c7411d4e004 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ re_chunk = { path = "crates/store/re_chunk", version = "=0.19.0-alpha.1", defaul re_chunk_store = { path = "crates/store/re_chunk_store", version = "=0.19.0-alpha.1", default-features = false } re_data_loader = { path = "crates/store/re_data_loader", version = "=0.19.0-alpha.1", default-features = false } re_data_source = { path = "crates/store/re_data_source", version = "=0.19.0-alpha.1", default-features = false } +re_dataframe = { path = "crates/store/re_dataframe", version = "=0.19.0-alpha.1", default-features = false } re_entity_db = { path = "crates/store/re_entity_db", version = "=0.19.0-alpha.1", default-features = false } re_format_arrow = { path = "crates/store/re_format_arrow", version = "=0.19.0-alpha.1", default-features = false } re_log_encoding = { path = "crates/store/re_log_encoding", version = "=0.19.0-alpha.1", default-features = false } @@ -513,12 +514,12 @@ missing_errors_doc = "allow" # As a last resport, patch with a commit to our own repository. # ALWAYS document what PR the commit hash is part of, or when it was merged into the upstream trunk. -ecolor = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" } # egui master 2024-09-03 -eframe = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" } # egui master 2024-09-03 -egui = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" } # egui master 2024-09-03 -egui_extras = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" } # egui master 2024-09-03 -egui-wgpu = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" } # egui master 2024-09-03 -emath = { git = "https://github.com/emilk/egui.git", rev = "454abf705b87aba70cef582d6ce80f74aa398906" } # egui master 2024-09-03 +ecolor = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06 +eframe = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06 +egui = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06 +egui_extras = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06 +egui-wgpu = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06 +emath = { git = "https://github.com/emilk/egui.git", rev = "6b7f4312373a301a4cdf7d99a0d546acd34bcd66" } # egui master 2024-09-06 # Useful while developing: # ecolor = { path = "../../egui/crates/ecolor" } diff --git a/crates/store/re_chunk/src/transport.rs b/crates/store/re_chunk/src/transport.rs index 80b6d288953c..bd5a31424aaa 100644 --- a/crates/store/re_chunk/src/transport.rs +++ b/crates/store/re_chunk/src/transport.rs @@ -273,6 +273,15 @@ impl TransportChunk { }) } + #[inline] + pub fn all_columns(&self) -> impl Iterator)> + '_ { + self.schema + .fields + .iter() + .enumerate() + .filter_map(|(i, field)| self.data.columns().get(i).map(|column| (field, column))) + } + /// Iterates all control columns present in this chunk. #[inline] pub fn controls(&self) -> impl Iterator)> { diff --git a/crates/store/re_chunk_store/src/dataframe.rs b/crates/store/re_chunk_store/src/dataframe.rs index 2b2bc6705703..ba5cd27bdddb 100644 --- a/crates/store/re_chunk_store/src/dataframe.rs +++ b/crates/store/re_chunk_store/src/dataframe.rs @@ -65,6 +65,15 @@ impl ColumnDescriptor { Self::Component(descr) => descr.to_arrow_field(), } } + + #[inline] + pub fn short_name(&self) -> String { + match self { + Self::Control(descr) => descr.component_name.short_name().to_owned(), + Self::Time(descr) => descr.timeline.name().to_string(), + Self::Component(descr) => descr.component_name.short_name().to_owned(), + } + } } /// Describes a column used to control Rerun's behavior, such as `RowId`. @@ -72,7 +81,7 @@ impl ColumnDescriptor { pub struct ControlColumnDescriptor { /// Semantic name associated with this data. /// - /// Example: `rerun.controls.RowId`. + /// Example: `RowId::name()`. pub component_name: ComponentName, /// The Arrow datatype of the column. diff --git a/crates/store/re_chunk_store/src/lib.rs b/crates/store/re_chunk_store/src/lib.rs index b04104564678..b3deadc40405 100644 --- a/crates/store/re_chunk_store/src/lib.rs +++ b/crates/store/re_chunk_store/src/lib.rs @@ -45,6 +45,8 @@ pub use re_log_encoding::decoder::VersionPolicy; pub use re_log_types::{ResolvedTimeRange, TimeInt, TimeType, Timeline}; pub mod external { + pub use arrow2; + pub use re_chunk; pub use re_log_encoding; } diff --git a/crates/store/re_dataframe/src/latest_at.rs b/crates/store/re_dataframe/src/latest_at.rs index 591ccc0dc8f9..3dee374aaf54 100644 --- a/crates/store/re_dataframe/src/latest_at.rs +++ b/crates/store/re_dataframe/src/latest_at.rs @@ -56,6 +56,11 @@ impl<'a> LatestAtQueryHandle<'a> { } impl LatestAtQueryHandle<'_> { + /// The query expression used to instantiate this handle. + pub fn query(&self) -> &LatestAtQueryExpression { + &self.query + } + /// All results returned by this handle will strictly follow this schema. /// /// Columns that do not yield any data will still be present in the results, filled with null values. diff --git a/crates/store/re_dataframe/src/range.rs b/crates/store/re_dataframe/src/range.rs index e56258bc01bc..0917d5dae6f7 100644 --- a/crates/store/re_dataframe/src/range.rs +++ b/crates/store/re_dataframe/src/range.rs @@ -133,6 +133,11 @@ impl RangeQueryHandle<'_> { }) } + /// The query used to instantiate this handle. + pub fn query(&self) -> &RangeQueryExpression { + &self.query + } + /// All results returned by this handle will strictly follow this schema. /// /// Columns that do not yield any data will still be present in the results, filled with null values. diff --git a/crates/store/re_entity_db/Cargo.toml b/crates/store/re_entity_db/Cargo.toml index 9b2f6e326543..ec35b345c900 100644 --- a/crates/store/re_entity_db/Cargo.toml +++ b/crates/store/re_entity_db/Cargo.toml @@ -30,6 +30,7 @@ serde = ["dep:serde", "re_log_types/serde"] re_build_info.workspace = true re_chunk = { workspace = true, features = ["serde"] } re_chunk_store.workspace = true +re_dataframe.workspace = true re_format.workspace = true re_int_histogram.workspace = true re_log.workspace = true diff --git a/crates/store/re_entity_db/src/entity_db.rs b/crates/store/re_entity_db/src/entity_db.rs index 1ef80e851195..7c6aeb4a548f 100644 --- a/crates/store/re_entity_db/src/entity_db.rs +++ b/crates/store/re_entity_db/src/entity_db.rs @@ -121,6 +121,13 @@ impl EntityDb { &self.query_caches } + pub fn query_engine(&self) -> re_dataframe::QueryEngine<'_> { + re_dataframe::QueryEngine { + store: self.store(), + cache: self.query_caches(), + } + } + /// Queries for the given `component_names` using latest-at semantics. /// /// See [`re_query::LatestAtResults`] for more information about how to handle the results. diff --git a/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs b/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs index a4e0a2f13067..be7e6bb798e0 100644 --- a/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs +++ b/crates/viewer/re_chunk_store_ui/src/arrow_ui.rs @@ -1,7 +1,7 @@ use itertools::Itertools; -use re_chunk_store::external::re_chunk::external::arrow2; -use re_chunk_store::external::re_chunk::external::arrow2::array::Utf8Array; +use re_chunk_store::external::arrow2; +use re_chunk_store::external::arrow2::array::Utf8Array; use re_types::SizeBytes as _; use re_ui::UiExt; diff --git a/crates/viewer/re_chunk_store_ui/src/chunk_list_mode.rs b/crates/viewer/re_chunk_store_ui/src/chunk_list_mode.rs index f8a27a465546..b4c9567c3345 100644 --- a/crates/viewer/re_chunk_store_ui/src/chunk_list_mode.rs +++ b/crates/viewer/re_chunk_store_ui/src/chunk_list_mode.rs @@ -70,7 +70,7 @@ impl ChunkListMode { .. } ), - "Latest at", + "Latest-at", ) .on_hover_text("Display chunks relevant to the provided latest-at query") .clicked() diff --git a/crates/viewer/re_chunk_store_ui/src/chunk_store_ui.rs b/crates/viewer/re_chunk_store_ui/src/chunk_store_ui.rs index 8cb6f78c2cf0..42ead613f0f3 100644 --- a/crates/viewer/re_chunk_store_ui/src/chunk_store_ui.rs +++ b/crates/viewer/re_chunk_store_ui/src/chunk_store_ui.rs @@ -334,8 +334,6 @@ impl DatastoreUi { ) .resizable(true) .vscroll(true) - //TODO(ab): remove when https://github.com/emilk/egui/pull/4817 is merged/released - .max_scroll_height(f32::INFINITY) .auto_shrink([false, false]) .striped(true); diff --git a/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs b/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs index 7c674fe1faf3..d7ed89f92bea 100644 --- a/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs +++ b/crates/viewer/re_chunk_store_ui/src/chunk_ui.rs @@ -171,8 +171,6 @@ impl ChunkUi { ) .resizable(true) .vscroll(true) - //TODO(ab): remove when https://github.com/emilk/egui/pull/4817 is merged/released - .max_scroll_height(f32::INFINITY) .auto_shrink([false, false]) .striped(true); diff --git a/crates/viewer/re_space_view_dataframe/Cargo.toml b/crates/viewer/re_space_view_dataframe/Cargo.toml index d8154b8bf3d8..a92540e3bb41 100644 --- a/crates/viewer/re_space_view_dataframe/Cargo.toml +++ b/crates/viewer/re_space_view_dataframe/Cargo.toml @@ -21,6 +21,7 @@ all-features = true [dependencies] re_chunk_store.workspace = true re_data_ui.workspace = true +re_dataframe.workspace = true re_entity_db.workspace = true re_log.workspace = true re_log_types.workspace = true @@ -33,6 +34,9 @@ re_ui.workspace = true re_viewer_context.workspace = true re_viewport_blueprint.workspace = true +anyhow.workspace = true egui_extras.workspace = true +egui_table = { git = "https://github.com/rerun-io/egui_table.git", rev = "0f594701d528c4a9553521cb941de1886549dc70" } # main as of 2024-09-09 egui.workspace = true itertools.workspace = true +thiserror.workspace = true diff --git a/crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs b/crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs new file mode 100644 index 000000000000..c4d0f397661b --- /dev/null +++ b/crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs @@ -0,0 +1,350 @@ +use std::collections::BTreeMap; +use std::ops::Range; + +use anyhow::Context; +use itertools::Itertools; + +use re_chunk_store::{ColumnDescriptor, LatestAtQuery, RowId}; +use re_dataframe::{LatestAtQueryHandle, RangeQueryHandle, RecordBatch}; +use re_log_types::{EntityPath, TimeInt, Timeline}; +use re_types_core::Loggable as _; +use re_ui::UiExt as _; +use re_viewer_context::ViewerContext; + +use crate::display_record_batch::{DisplayRecordBatch, DisplayRecordBatchError}; + +/// Display a dataframe table for the provided query. +pub(crate) fn dataframe_ui<'a>( + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + query: impl Into>, +) { + dataframe_ui_impl(ctx, ui, &query.into()); +} + +/// A query handle for either a latest-at or range query. +pub(crate) enum QueryHandle<'a> { + LatestAt(LatestAtQueryHandle<'a>), + Range(RangeQueryHandle<'a>), +} + +impl QueryHandle<'_> { + fn schema(&self) -> &[ColumnDescriptor] { + match self { + QueryHandle::LatestAt(query_handle) => query_handle.schema(), + QueryHandle::Range(query_handle) => query_handle.schema(), + } + } + + fn num_rows(&self) -> u64 { + match self { + QueryHandle::LatestAt(_) => 1, + QueryHandle::Range(query_handle) => query_handle.num_rows(), + } + } + + fn get(&self, start: u64, num_rows: u64) -> Vec { + match self { + QueryHandle::LatestAt(query_handle) => { + // latest-at queries only have one row + debug_assert_eq!((start, num_rows), (0, 1)); + + vec![query_handle.get()] + } + QueryHandle::Range(query_handle) => query_handle.get(start, num_rows), + } + } + + fn timeline(&self) -> Timeline { + match self { + QueryHandle::LatestAt(query_handle) => query_handle.query().timeline, + QueryHandle::Range(query_handle) => query_handle.query().timeline, + } + } +} + +impl<'a> From> for QueryHandle<'a> { + fn from(query_handle: LatestAtQueryHandle<'a>) -> Self { + QueryHandle::LatestAt(query_handle) + } +} + +impl<'a> From> for QueryHandle<'a> { + fn from(query_handle: RangeQueryHandle<'a>) -> Self { + QueryHandle::Range(query_handle) + } +} + +#[derive(Clone, Copy)] +struct BatchRef { + /// Which batch? + batch_idx: usize, + + /// Which row within the batch? + row_idx: usize, +} + +/// This structure maintains the data for displaying rows in a table. +/// +/// Row data is stored in a bunch of [`DisplayRecordBatch`], which are created from +/// [`RecordBatch`]s. We also maintain a mapping for each row number to the corresponding record +/// batch and the index inside it. +struct RowsDisplayData { + /// The [`DisplayRecordBatch`]s to display. + display_record_batches: Vec, + + /// For each row to be displayed, where can we find the data? + batch_ref_from_row: BTreeMap, + + /// The index of the time column corresponding to the query timeline. + query_time_column_index: Option, + + /// The index of the time column corresponding the row IDs. + row_id_column_index: Option, +} + +impl RowsDisplayData { + fn try_new( + row_indices: &Range, + record_batches: Vec, + schema: &[ColumnDescriptor], + query_timeline: &Timeline, + ) -> Result { + let display_record_batches = record_batches + .into_iter() + .map(|record_batch| DisplayRecordBatch::try_new(&record_batch, schema)) + .collect::, _>>()?; + + let mut batch_ref_from_row = BTreeMap::new(); + let mut offset = row_indices.start; + for (batch_idx, batch) in display_record_batches.iter().enumerate() { + let batch_len = batch.num_rows(); + for row_idx in 0..batch_len { + batch_ref_from_row.insert(offset + row_idx as u64, BatchRef { batch_idx, row_idx }); + } + offset += batch_len as u64; + } + + // find the time column + let query_time_column_index = schema + .iter() + .find_position(|desc| match desc { + ColumnDescriptor::Time(time_column_desc) => { + &time_column_desc.timeline == query_timeline + } + _ => false, + }) + .map(|(pos, _)| pos); + + // find the row id column + let row_id_column_index = schema + .iter() + .find_position(|desc| match desc { + ColumnDescriptor::Control(control_column_desc) => { + control_column_desc.component_name == RowId::name() + } + _ => false, + }) + .map(|(pos, _)| pos); + + Ok(Self { + display_record_batches, + batch_ref_from_row, + query_time_column_index, + row_id_column_index, + }) + } +} + +/// [`egui_table::TableDelegate`] implementation for displaying a [`QueryHandle`] in a table. +struct DataframeTableDelegate<'a> { + ctx: &'a ViewerContext<'a>, + query_handle: &'a QueryHandle<'a>, + schema: &'a [ColumnDescriptor], + header_entity_paths: Vec>, + display_data: anyhow::Result, + + num_rows: u64, +} + +impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> { + fn prepare(&mut self, info: &egui_table::PrefetchInfo) { + re_tracing::profile_function!(); + + let data = RowsDisplayData::try_new( + &info.visible_rows, + self.query_handle.get( + info.visible_rows.start, + info.visible_rows.end - info.visible_rows.start, + ), + self.schema, + &self.query_handle.timeline(), + ); + + self.display_data = data.context("Failed to create display data"); + } + + fn header_cell_ui(&mut self, ui: &mut egui::Ui, cell: &egui_table::HeaderCellInfo) { + egui::Frame::none() + .inner_margin(egui::Margin::symmetric(4.0, 0.0)) + .show(ui, |ui| { + if cell.row_nr == 0 { + if let Some(entity_path) = &self.header_entity_paths[cell.group_index] { + ui.label(entity_path.to_string()); + } + } else if cell.row_nr == 1 { + ui.strong(self.schema[cell.col_range.start].short_name()); + } else { + // this should never happen + error_ui(ui, format!("Unexpected header row_nr: {}", cell.row_nr)); + } + }); + } + + fn cell_ui(&mut self, ui: &mut egui::Ui, cell: &egui_table::CellInfo) { + re_tracing::profile_function!(); + + if cell.row_nr % 2 == 1 { + // Paint stripes + ui.painter() + .rect_filled(ui.max_rect(), 0.0, ui.visuals().faint_bg_color); + } + + debug_assert!(cell.row_nr < self.num_rows, "Bug in egui_table"); + + let display_data = match &self.display_data { + Ok(display_data) => display_data, + Err(err) => { + error_ui(ui, format!("Error with display data: {err}")); + return; + } + }; + + let cell_ui = |ui: &mut egui::Ui| { + if let Some(BatchRef { batch_idx, row_idx }) = + display_data.batch_ref_from_row.get(&cell.row_nr).copied() + { + let batch = &display_data.display_record_batches[batch_idx]; + let column = &batch.columns()[cell.col_nr]; + + // compute the latest-at query for this row (used to display tooltips) + let timestamp = display_data + .query_time_column_index + .and_then(|col_idx| { + display_data.display_record_batches[batch_idx].columns()[col_idx] + .try_decode_time(row_idx) + }) + .unwrap_or(TimeInt::MAX); + let latest_at_query = LatestAtQuery::new(self.query_handle.timeline(), timestamp); + let row_id = display_data + .row_id_column_index + .and_then(|col_idx| { + display_data.display_record_batches[batch_idx].columns()[col_idx] + .try_decode_row_id(row_idx) + }) + .unwrap_or(RowId::ZERO); + + if ui.is_sizing_pass() { + ui.style_mut().wrap_mode = Some(egui::TextWrapMode::Extend); + } else { + ui.style_mut().wrap_mode = Some(egui::TextWrapMode::Truncate); + } + column.data_ui(self.ctx, ui, row_id, &latest_at_query, row_idx); + } else { + error_ui( + ui, + "Bug in egui_table: we didn't prefetch what was rendered!", + ); + } + }; + + egui::Frame::none() + .inner_margin(egui::Margin::symmetric(4.0, 0.0)) + .show(ui, cell_ui); + } +} + +/// Display the result of a [`QueryHandle`] in a table. +fn dataframe_ui_impl(ctx: &ViewerContext<'_>, ui: &mut egui::Ui, query_handle: &QueryHandle<'_>) { + re_tracing::profile_function!(); + + let schema = query_handle.schema(); + let (header_groups, header_entity_paths) = column_groups_for_entity(schema); + + let num_rows = query_handle.num_rows(); + + let mut table_delegate = DataframeTableDelegate { + ctx, + query_handle, + schema, + header_entity_paths, + num_rows, + display_data: Err(anyhow::anyhow!( + "No row data, `fetch_columns_and_rows` not called." + )), + }; + + let num_sticky_cols = schema + .iter() + .take_while(|cd| matches!(cd, ColumnDescriptor::Control(_) | ColumnDescriptor::Time(_))) + .count(); + + egui::Frame::none().inner_margin(5.0).show(ui, |ui| { + egui_table::Table::new() + .columns( + schema + .iter() + .map(|column_descr| { + egui_table::Column::new(200.0) + .resizable(true) + .id(egui::Id::new(column_descr)) + }) + .collect::>(), + ) + .num_sticky_cols(num_sticky_cols) + .headers(vec![ + egui_table::HeaderRow { + height: re_ui::DesignTokens::table_header_height(), + groups: header_groups, + }, + egui_table::HeaderRow::new(re_ui::DesignTokens::table_header_height()), + ]) + .num_rows(num_rows) + .row_height(re_ui::DesignTokens::table_line_height()) + .show(ui, &mut table_delegate); + }); +} + +/// Groups column by entity paths. +fn column_groups_for_entity( + columns: &[ColumnDescriptor], +) -> (Vec>, Vec>) { + if columns.is_empty() { + (vec![], vec![]) + } else if columns.len() == 1 { + #[allow(clippy::single_range_in_vec_init)] + (vec![0..1], vec![columns[0].entity_path().cloned()]) + } else { + let mut groups = vec![]; + let mut entity_paths = vec![]; + let mut start = 0; + let mut current_entity = columns[0].entity_path(); + for (i, column) in columns.iter().enumerate().skip(1) { + if column.entity_path() != current_entity { + groups.push(start..i); + entity_paths.push(current_entity.cloned()); + start = i; + current_entity = column.entity_path(); + } + } + groups.push(start..columns.len()); + entity_paths.push(current_entity.cloned()); + (groups, entity_paths) + } +} + +fn error_ui(ui: &mut egui::Ui, error: impl AsRef) { + let error = error.as_ref(); + ui.error_label(error); + re_log::warn_once!("{error}"); +} diff --git a/crates/viewer/re_space_view_dataframe/src/display_record_batch.rs b/crates/viewer/re_space_view_dataframe/src/display_record_batch.rs new file mode 100644 index 000000000000..98cb31e59e3a --- /dev/null +++ b/crates/viewer/re_space_view_dataframe/src/display_record_batch.rs @@ -0,0 +1,344 @@ +//! Intermediate data structures to make `re_datastore`'s schemas and [`RecordBatch`]s more amenable +//! to displaying in a table. + +use thiserror::Error; + +use re_chunk_store::external::arrow2::{ + array::{ + Array as ArrowArray, DictionaryArray as ArrowDictionaryArray, ListArray as ArrowListArray, + PrimitiveArray as ArrowPrimitiveArray, StructArray as ArrowStructArray, + }, + datatypes::DataType, + datatypes::DataType as ArrowDataType, +}; +use re_chunk_store::{ColumnDescriptor, ComponentColumnDescriptor, LatestAtQuery, RowId}; +use re_dataframe::RecordBatch; +use re_log_types::{EntityPath, TimeInt, TimeType, Timeline}; +use re_types::external::arrow2::datatypes::IntegerType; +use re_types_core::{ComponentName, Loggable as _}; +use re_ui::UiExt; +use re_viewer_context::{UiLayout, ViewerContext}; + +#[derive(Error, Debug)] +pub(crate) enum DisplayRecordBatchError { + #[error("Unknown control column: {0}")] + UnknownControlColumn(String), + + #[error("Unexpected column data type for timeline '{0}': {1:?}")] + UnexpectedTimeColumnDataType(String, ArrowDataType), + + #[error("Unexpected column data type for component '{0}': {1:?}")] + UnexpectedComponentColumnDataType(String, ArrowDataType), +} + +pub(crate) enum ComponentData { + Null, + ListArray(ArrowListArray), + DictionaryArray { + dict: ArrowDictionaryArray, + values: ArrowListArray, + }, +} + +impl ComponentData { + #[allow(clippy::borrowed_box)] // https://github.com/rust-lang/rust-clippy/issues/11940 + fn try_new( + descriptor: &ComponentColumnDescriptor, + column_data: &Box, + ) -> Result { + match column_data.data_type() { + DataType::Null => Ok(Self::Null), + DataType::List(_) => Ok(Self::ListArray( + column_data + .as_any() + .downcast_ref::>() + .expect("`data_type` checked, failure is a bug in re_dataframe") + .clone(), + )), + DataType::Dictionary(IntegerType::Int32, _, _) => { + let dict = column_data + .as_any() + .downcast_ref::>() + .expect("`data_type` checked, failure is a bug in re_dataframe") + .clone(); + let values = dict + .values() + .as_any() + .downcast_ref::>() + .expect("`data_type` checked, failure is a bug in re_dataframe") + .clone(); + Ok(Self::DictionaryArray { dict, values }) + } + _ => Err(DisplayRecordBatchError::UnexpectedComponentColumnDataType( + descriptor.component_name.to_string(), + column_data.data_type().to_owned(), + )), + } + } + + #[allow(clippy::too_many_arguments)] + fn data_ui( + &self, + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + row_id: RowId, + latest_at_query: &LatestAtQuery, + entity_path: &EntityPath, + component_name: ComponentName, + row_index: usize, // index within the batch column + ) { + let data = match self { + Self::Null => { + ui.label("null"); + return; + } + Self::ListArray(list_array) => list_array + .is_valid(row_index) + .then(|| list_array.value(row_index)), + Self::DictionaryArray { dict, values } => dict + .is_valid(row_index) + .then(|| values.value(dict.key_value(row_index))), + }; + + if let Some(data) = data { + ctx.component_ui_registry.ui_raw( + ctx, + ui, + UiLayout::List, + latest_at_query, + ctx.recording(), + entity_path, + component_name, + Some(row_id), + &*data, + ); + } else { + ui.label("-"); + } + } +} + +pub(crate) enum DisplayColumn { + RowId { + row_id_times: ArrowPrimitiveArray, + row_id_counters: ArrowPrimitiveArray, + }, + Timeline { + timeline: Timeline, + time_data: ArrowPrimitiveArray, + }, + Component { + entity_path: EntityPath, + component_name: ComponentName, + component_data: ComponentData, + }, +} + +impl DisplayColumn { + #[allow(clippy::borrowed_box)] // https://github.com/rust-lang/rust-clippy/issues/11940 + fn try_new( + column_schema: &ColumnDescriptor, + column_data: &Box, + ) -> Result { + match column_schema { + ColumnDescriptor::Control(desc) => { + if desc.component_name == RowId::name() { + let row_ids = column_data + .as_any() + .downcast_ref::() + .expect("expected format for RowId, failure is a bug in re_dataframe"); + let [times, counters] = row_ids.values() else { + panic!("RowIds are corrupt -- this should be impossible (sanity checked)"); + }; + + #[allow(clippy::unwrap_used)] + let row_id_times = times + .as_any() + .downcast_ref::>() + .expect("expected format for RowId, failure is a bug in re_dataframe") + .clone(); + + #[allow(clippy::unwrap_used)] + let row_id_counters = counters + .as_any() + .downcast_ref::>() + .expect("expected format for RowId, failure is a bug in re_dataframe") + .clone(); + + Ok(Self::RowId { + //descriptor: desc, + row_id_times, + row_id_counters, + }) + } else { + Err(DisplayRecordBatchError::UnknownControlColumn( + desc.component_name.to_string(), + )) + } + } + ColumnDescriptor::Time(desc) => { + let time_data = column_data + .as_any() + .downcast_ref::>() + .ok_or_else(|| { + DisplayRecordBatchError::UnexpectedTimeColumnDataType( + desc.timeline.name().as_str().to_owned(), + column_data.data_type().to_owned(), + ) + })? + .clone(); + + Ok(Self::Timeline { + timeline: desc.timeline, + time_data, + }) + } + ColumnDescriptor::Component(desc) => Ok(Self::Component { + entity_path: desc.entity_path.clone(), + component_name: desc.component_name, + component_data: ComponentData::try_new(desc, column_data)?, + }), + } + } + + pub(crate) fn data_ui( + &self, + ctx: &ViewerContext<'_>, + ui: &mut egui::Ui, + row_id: RowId, + latest_at_query: &LatestAtQuery, + index: usize, + ) { + match self { + Self::RowId { + row_id_times, + row_id_counters, + .. + } => { + let row_id = RowId::from_u128( + (row_id_times.value(index) as u128) << 64 + | (row_id_counters.value(index) as u128), + ); + row_id_ui(ctx, ui, &row_id); + } + Self::Timeline { + timeline, + time_data, + } => { + let timestamp = TimeInt::try_from(time_data.value(index)); + match timestamp { + Ok(timestamp) => { + ui.label(timeline.typ().format(timestamp, ctx.app_options.time_zone)); + } + Err(err) => { + ui.error_label(&format!("{err}")); + } + } + } + Self::Component { + entity_path, + component_name, + component_data, + } => { + component_data.data_ui( + ctx, + ui, + row_id, + latest_at_query, + entity_path, + *component_name, + index, + ); + } + } + } + + /// Try to decode the row ID from the given row index. + /// + /// Succeeds only if the column is a `RowId` column. + pub(crate) fn try_decode_row_id(&self, row_index: usize) -> Option { + match self { + Self::RowId { + row_id_times, + row_id_counters, + } => { + let time = row_id_times.value(row_index); + let counter = row_id_counters.value(row_index); + Some(RowId::from_u128((time as u128) << 64 | (counter as u128))) + } + _ => None, + } + } + + /// Try to decode the time from the given row index. + /// + /// Succeeds only if the column is a `Timeline` column. + pub(crate) fn try_decode_time(&self, row_index: usize) -> Option { + match self { + Self::Timeline { time_data, .. } => { + let timestamp = time_data.value(row_index); + TimeInt::try_from(timestamp).ok() + } + _ => None, + } + } +} + +pub(crate) struct DisplayRecordBatch { + num_rows: usize, + columns: Vec, +} + +impl DisplayRecordBatch { + /// Create a new `DisplayRecordBatch` from a `RecordBatch` and its schema. + /// + /// The columns in the record batch must match the schema. This is guaranteed by `re_datastore`. + pub(crate) fn try_new( + record_batch: &RecordBatch, + schema: &[ColumnDescriptor], + ) -> Result { + let columns: Result, _> = schema + .iter() + .zip(record_batch.all_columns()) + .map(|(column_schema, (_, column_data))| { + DisplayColumn::try_new(column_schema, column_data) + }) + .collect(); + + Ok(Self { + num_rows: record_batch.num_rows(), + columns: columns?, + }) + } + + pub(crate) fn num_rows(&self) -> usize { + self.num_rows + } + + pub(crate) fn columns(&self) -> &[DisplayColumn] { + &self.columns + } +} + +fn row_id_ui(ctx: &ViewerContext<'_>, ui: &mut egui::Ui, row_id: &RowId) { + let s = row_id.to_string(); + let split_pos = s.char_indices().nth_back(5); + + ui.label(match split_pos { + Some((pos, _)) => &s[pos..], + None => &s, + }) + .on_hover_ui(|ui| { + let text = format!( + "{}\n\nTimestamp: {}\nIncrement: {}", + s, + (row_id.nanoseconds_since_epoch() as i64) + .try_into() + .map(|t| TimeType::Time.format(TimeInt::from_nanos(t), ctx.app_options.time_zone)) + .unwrap_or("error decoding timestamp".to_owned()), + row_id.inc() + ); + + ui.label(text); + }); +} diff --git a/crates/viewer/re_space_view_dataframe/src/latest_at_table.rs b/crates/viewer/re_space_view_dataframe/src/latest_at_table.rs deleted file mode 100644 index 18fdd9cabdd7..000000000000 --- a/crates/viewer/re_space_view_dataframe/src/latest_at_table.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::collections::BTreeSet; - -use re_chunk_store::LatestAtQuery; -use re_data_ui::item_ui::instance_path_button; -use re_entity_db::InstancePath; -use re_log_types::Instance; -use re_viewer_context::{Item, UiLayout, ViewQuery, ViewerContext}; - -use crate::{ - table_ui::table_ui, - utils::{sorted_instance_paths_for, sorted_visible_entity_path}, -}; - -/// Display a "latest-at" table. -/// -/// This table has entity instances as rows and components as columns. That data is the result of a -/// "latest-at" query based on the current timeline and time. -pub(crate) fn latest_at_table_ui( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - query: &ViewQuery<'_>, - latest_at_query: &LatestAtQuery, -) { - re_tracing::profile_function!(); - - // - // DATA - // - - // These are the entity paths whose content we must display. - let sorted_entity_paths = sorted_visible_entity_path(ctx, query); - - let sorted_instance_paths: Vec<_>; - let sorted_components: BTreeSet<_>; - { - re_tracing::profile_scope!("query"); - - // Produce a sorted list of each entity with all their instance keys. This will be the rows - // of the table. - // - // Important: our semantics here differs from other built-in space views. "Out-of-bound" - // instance keys (aka instance keys from a secondary component that cannot be joined with a - // primary component) are not filtered out. Reasons: - // - Primary/secondary component distinction only makes sense with archetypes, which we - // ignore. TODO(#4466): make archetypes more explicit? - // - This space view is about showing all user data anyways. - // - // Note: this must be a `Vec<_>` because we need random access for `body.rows()`. - sorted_instance_paths = sorted_entity_paths - .iter() - .flat_map(|entity_path| { - sorted_instance_paths_for( - entity_path, - ctx.recording_store(), - &latest_at_query.timeline(), - latest_at_query, - ) - }) - .collect(); - - // Produce a sorted list of all components that are present in one or more entities. These - // will be the columns of the table. - sorted_components = sorted_entity_paths - .iter() - .flat_map(|entity_path| { - ctx.recording_store() - .all_components_on_timeline(&latest_at_query.timeline(), entity_path) - .unwrap_or_default() - }) - // TODO(#4466): make showing/hiding indicators components an explicit optional - .filter(|comp| !comp.is_indicator_component()) - .collect(); - } - - // - // SCROLL TO ROW - // - - let index_for_instance_path = |instance_path: &InstancePath| { - let instance_path = if instance_path.instance == Instance::ALL { - InstancePath::instance(instance_path.entity_path.clone(), 0.into()) - } else { - instance_path.clone() - }; - - sorted_instance_paths.binary_search(&instance_path).ok() - }; - - let scroll_to_row = ctx.focused_item.as_ref().and_then(|item| match item { - Item::AppId(_) - | Item::DataSource(_) - | Item::StoreId(_) - | Item::ComponentPath(_) //TODO(ab): implement scroll to column? - | Item::SpaceView(_) - | Item::Container(_) => None, - - Item::InstancePath(instance_path) => index_for_instance_path(instance_path), - Item::DataResult(space_view_id, instance_path) => { - // We want to scroll only if the focus action originated from outside the table. We - // allow the case of `Instance::ALL` for when the entity is double-clicked in the - // blueprint tree. - //TODO(#6906): we should have an explicit way to track the "source" of the focus event. - let should_scroll = - (space_view_id != &query.space_view_id) || instance_path.instance.is_all(); - - should_scroll - .then(|| index_for_instance_path(instance_path)) - .flatten() - } - }); - - // - // DRAW - // - - // Draw the header row. - let header_ui = |mut row: egui_extras::TableRow<'_, '_>| { - row.col(|ui| { - ui.strong("Entity"); - }); - - for comp in &sorted_components { - row.col(|ui| { - ui.strong(comp.short_name()); - }); - } - }; - - // Draw a single line of the table. This is called for each _visible_ row, so it's ok to - // duplicate some of the querying. - let row_ui = |mut row: egui_extras::TableRow<'_, '_>| { - let instance_path = &sorted_instance_paths[row.index()]; - - row.col(|ui| { - instance_path_button( - ctx, - latest_at_query, - ctx.recording(), - ui, - Some(query.space_view_id), - instance_path, - ); - }); - - // Note: a lot of duplicated querying potentially happens here, but this is ok since this - // code runs *only* for visible rows. - for component_name in &sorted_components { - row.col(|ui| { - // TODO(ab, cmc): use the suitable API from re_query when it becomes available. - - let result = ctx - .recording_store() - .latest_at_relevant_chunks( - latest_at_query, - &instance_path.entity_path, - *component_name, - ) - .into_iter() - .filter_map(|chunk| { - let (index, unit) = chunk - .latest_at(latest_at_query, *component_name) - .into_unit() - .and_then(|unit| { - unit.index(&latest_at_query.timeline()) - .map(|index| (index, unit)) - })?; - - unit.component_batch_raw(component_name) - .map(|array| (index, array)) - }) - .max_by_key(|(index, _array)| *index); - - // TODO(#4466): it would be nice to display the time and row id somewhere, since we - // have them. - if let Some(((_time, row_id), array)) = result { - let instance_index = instance_path.instance.get() as usize; - - if array.is_empty() { - ui.weak("-"); - } else { - let (data, clamped) = if instance_index >= array.len() { - (array.sliced(array.len() - 1, 1), true) - } else { - (array.sliced(instance_index, 1), false) - }; - - ui.add_enabled_ui(!clamped, |ui| { - ctx.component_ui_registry.ui_raw( - ctx, - ui, - UiLayout::List, - latest_at_query, - ctx.recording(), - &instance_path.entity_path, - *component_name, - Some(row_id), - &*data, - ); - }); - } - } else { - ui.weak("-"); - } - }); - } - }; - - table_ui( - ui, - &sorted_components, - 1, // entity column - header_ui, - sorted_instance_paths.len(), - row_ui, - scroll_to_row, - ); -} diff --git a/crates/viewer/re_space_view_dataframe/src/lib.rs b/crates/viewer/re_space_view_dataframe/src/lib.rs index 92e83e1e8ed0..f7093d3a94dc 100644 --- a/crates/viewer/re_space_view_dataframe/src/lib.rs +++ b/crates/viewer/re_space_view_dataframe/src/lib.rs @@ -2,12 +2,10 @@ //! //! A Space View that shows the data contained in entities in a table. -mod latest_at_table; +mod dataframe_ui; +mod display_record_batch; mod query_kind; mod space_view_class; -mod table_ui; -mod time_range_table; -mod utils; mod view_query; mod visualizer_system; diff --git a/crates/viewer/re_space_view_dataframe/src/space_view_class.rs b/crates/viewer/re_space_view_dataframe/src/space_view_class.rs index 0454eb7591d4..4679c3bc18bb 100644 --- a/crates/viewer/re_space_view_dataframe/src/space_view_class.rs +++ b/crates/viewer/re_space_view_dataframe/src/space_view_class.rs @@ -1,21 +1,17 @@ use egui::Ui; -use re_chunk_store::LatestAtQuery; -use re_log_types::{EntityPath, ResolvedTimeRange}; +use crate::dataframe_ui::dataframe_ui; +use crate::{query_kind::QueryKind, view_query::Query, visualizer_system::EmptySystem}; +use re_log_types::{EntityPath, EntityPathFilter, ResolvedTimeRange}; use re_space_view::view_property_ui; -use re_types::blueprint::{archetypes, components}; +use re_types::blueprint::archetypes; use re_types_core::SpaceViewClassIdentifier; use re_ui::list_item; use re_viewer_context::{ SpaceViewClass, SpaceViewClassRegistryError, SpaceViewId, SpaceViewState, SpaceViewSystemExecutionError, SystemExecutionOutput, ViewQuery, ViewerContext, }; -use re_viewport_blueprint::ViewProperty; - -use crate::{ - latest_at_table::latest_at_table_ui, query_kind::QueryKind, - time_range_table::time_range_table_ui, view_query::Query, visualizer_system::EmptySystem, -}; +use re_viewport_blueprint::SpaceViewContents; #[derive(Default)] pub struct DataframeSpaceView; @@ -116,7 +112,7 @@ mode sets the default time range to _everything_. You can override this in the s &self, ctx: &ViewerContext<'_>, ui: &mut egui::Ui, - state: &mut dyn SpaceViewState, + _state: &mut dyn SpaceViewState, query: &ViewQuery<'_>, _system_output: SystemExecutionOutput, ) -> Result<(), SpaceViewSystemExecutionError> { @@ -136,42 +132,72 @@ mode sets the default time range to _everything_. You can override this in the s return Ok(()); }; + let query_engine = ctx.recording().query_engine(); + + let entity_path_filter = + Self::entity_path_filter(ctx, query.space_view_id, query.space_origin); + match query_mode { QueryKind::LatestAt { time } => { - latest_at_table_ui(ctx, ui, query, &LatestAtQuery::new(*timeline, time)); + let query = re_chunk_store::LatestAtQueryExpression { + entity_path_filter, + timeline: *timeline, + at: time, + }; + + //TODO(ab): specify which columns + let query_handle = query_engine.latest_at(&query, None); + + dataframe_ui(ctx, ui, query_handle); } QueryKind::Range { - pov_entity: _pov_entity, - pov_component: _pov_component, + pov_entity, + pov_component, from, to, } => { - //TODO(#7279): use pov entity and component - let time_range_table_order = - ViewProperty::from_archetype::( - ctx.blueprint_db(), - ctx.blueprint_query, - query.space_view_id, - ); - let sort_key = time_range_table_order - .component_or_fallback::(ctx, self, state)?; - let sort_order = time_range_table_order - .component_or_fallback::(ctx, self, state)?; - - time_range_table_ui( - ctx, - ui, - query, - sort_key, - sort_order, - timeline, - ResolvedTimeRange::new(from, to), - ); + let query = re_chunk_store::RangeQueryExpression { + entity_path_filter, + timeline: *timeline, + time_range: ResolvedTimeRange::new(from, to), + //TODO(#7365): using ComponentColumnDescriptor to specify PoV needs to go + pov: re_chunk_store::ComponentColumnDescriptor { + entity_path: pov_entity.clone(), + archetype_name: None, + archetype_field_name: None, + component_name: pov_component, + // this is actually ignored: + datatype: re_chunk_store::external::arrow2::datatypes::DataType::Null, + is_static: false, + }, + }; + + //TODO(ab): specify which columns should be displayed or not + dataframe_ui(ctx, ui, query_engine.range(&query, None)); } - } + }; Ok(()) } } +impl DataframeSpaceView { + fn entity_path_filter( + ctx: &ViewerContext<'_>, + space_view_id: SpaceViewId, + space_origin: &EntityPath, + ) -> EntityPathFilter { + //TODO(ab): this feels a little bit hacky but there isn't currently another way to get to + //the original entity path filter. + SpaceViewContents::from_db_or_default( + space_view_id, + ctx.blueprint_db(), + ctx.blueprint_query, + Self::identifier(), + &re_log_types::EntityPathSubs::new_with_origin(space_origin), + ) + .entity_path_filter + } +} + re_viewer_context::impl_component_fallback_provider!(DataframeSpaceView => []); diff --git a/crates/viewer/re_space_view_dataframe/src/table_ui.rs b/crates/viewer/re_space_view_dataframe/src/table_ui.rs deleted file mode 100644 index 8b77c0e29aba..000000000000 --- a/crates/viewer/re_space_view_dataframe/src/table_ui.rs +++ /dev/null @@ -1,67 +0,0 @@ -use std::collections::BTreeSet; - -use egui_extras::{Column, TableRow}; - -use re_chunk_store::RowId; -use re_types_core::ComponentName; - -/// Display a nicely configured table with the provided header ui, row ui, and row count. -/// -/// The `extra_columns` are how many more columns there are in addition to the components. -pub(crate) fn table_ui( - ui: &mut egui::Ui, - sorted_components: &BTreeSet, - extra_columns: usize, - header_ui: impl FnOnce(egui_extras::TableRow<'_, '_>), - row_count: usize, - row_ui: impl FnMut(TableRow<'_, '_>), - scroll_to_row: Option, -) { - re_tracing::profile_function!(); - - egui::ScrollArea::horizontal() - .auto_shrink([false, false]) - .show(ui, |ui| { - ui.style_mut().wrap_mode = Some(egui::TextWrapMode::Extend); - - egui::Frame { - inner_margin: egui::Margin::same(5.0), - ..Default::default() - } - .show(ui, |ui| { - let mut table_builder = egui_extras::TableBuilder::new(ui) - .columns( - Column::auto_with_initial_suggestion(200.0).clip(true), - extra_columns + sorted_components.len(), - ) - .resizable(true) - .vscroll(true) - //TODO(ab): remove when https://github.com/emilk/egui/pull/4817 is merged/released - .max_scroll_height(f32::INFINITY) - .auto_shrink([false, false]) - .striped(true); - - if let Some(scroll_to_row) = scroll_to_row { - table_builder = - table_builder.scroll_to_row(scroll_to_row, Some(egui::Align::TOP)); - } - - table_builder - .header(re_ui::DesignTokens::table_line_height(), header_ui) - .body(|body| { - body.rows(re_ui::DesignTokens::table_line_height(), row_count, row_ui); - }); - }); - }); -} - -pub(crate) fn row_id_ui(ui: &mut egui::Ui, row_id: &RowId) { - let s = row_id.to_string(); - let split_pos = s.char_indices().nth_back(5); - - ui.label(match split_pos { - Some((pos, _)) => &s[pos..], - None => &s, - }) - .on_hover_text(s); -} diff --git a/crates/viewer/re_space_view_dataframe/src/time_range_table.rs b/crates/viewer/re_space_view_dataframe/src/time_range_table.rs deleted file mode 100644 index 623a6d7d5fd0..000000000000 --- a/crates/viewer/re_space_view_dataframe/src/time_range_table.rs +++ /dev/null @@ -1,288 +0,0 @@ -use std::collections::{BTreeMap, BTreeSet}; -use std::sync::Arc; - -use re_chunk_store::{Chunk, LatestAtQuery, RangeQuery, RowId}; -use re_data_ui::item_ui::entity_path_button; -use re_entity_db::InstancePath; -use re_log_types::{EntityPath, ResolvedTimeRange, TimeInt, Timeline}; -use re_types::blueprint::components::{SortKey, SortOrder}; -use re_viewer_context::{Item, UiLayout, ViewQuery, ViewerContext}; - -use crate::table_ui::{row_id_ui, table_ui}; - -/// Show a table with entities and time as rows, and components as columns. -/// -/// Here, a "row" is a tuple of `(entity_path, time, row_id)`. This means that both "over logging" -/// (i.e. logging multiple times the same entity/component at the same timestamp) and "split -/// logging" (i.e. multiple log calls on the same [entity, time] but with different components) -/// lead to multiple rows. In other words, no joining is done here. -/// -/// Also: -/// - View entities have their query range "forced" to a range query. If set to "latest-at", they -/// are converted to Rel(0)-Rel(0). -/// - Only the data logged in the query range is displayed. There is no implicit "latest-at" like -/// it's done for regular views. -/// - Static data is always shown. -/// - When both static and non-static data exist for the same entity/component, the non-static data -/// is never shown (as per our data model). -pub(crate) fn time_range_table_ui( - ctx: &ViewerContext<'_>, - ui: &mut egui::Ui, - query: &ViewQuery<'_>, - sort_key: SortKey, - sort_order: SortOrder, - timeline: &Timeline, - resolved_time_range: ResolvedTimeRange, -) { - re_tracing::profile_function!(); - - // - // Produce a sorted list of all components we are interested id. - // - - // TODO(ab): add support for filtering components more narrowly. - let sorted_components: BTreeSet<_> = { - re_tracing::profile_scope!("query components"); - - // Produce a sorted list of all components that are present in one or more entities. - // These will be the columns of the table. - query - .iter_all_data_results() - .filter(|data_result| data_result.is_visible(ctx)) - .flat_map(|data_result| { - ctx.recording_store() - .all_components_on_timeline(timeline, &data_result.entity_path) - .unwrap_or_default() - }) - // TODO(#4466): make showing/hiding indicators components an explicit optional - .filter(|comp| !comp.is_indicator_component()) - .collect() - }; - - // - // Build the full list of rows, along with the chunk where the data is. Rows are keyed by an - // (entity, time, row_id) tuple (see function docstring). These keys are mapped to the - // corresponding chunk that contains the actual data. - // - // We build a big, monolithic iterator for all the rows. The following code builds that by - // breaking it into several functions for sanity purpose, from innermost to outermost. - // - - type RowKey = (EntityPath, TimeInt, RowId); - - #[inline] - fn map_chunk_indices_to_key_value_iter<'a>( - indices_iter: impl Iterator + 'a, - chunk: Arc, - entity_path: EntityPath, - resolved_time_range: ResolvedTimeRange, - ) -> impl Iterator)> + 'a { - indices_iter - .filter(move |(time, _)| time.is_static() || resolved_time_range.contains(*time)) - .map(move |(time, row_id)| { - let chunk = chunk.clone(); - ((entity_path.clone(), time, row_id), chunk) - }) - } - - #[inline] - fn entity_to_key_value_iter<'a>( - ctx: &ViewerContext<'_>, - entity_path: &'a EntityPath, - timeline: Timeline, - resolved_time_range: ResolvedTimeRange, - ) -> impl Iterator)> + 'a { - let range_query = RangeQuery::new(timeline, resolved_time_range); - - ctx.recording_store() - .range_relevant_chunks_for_all_components(&range_query, entity_path) - .into_iter() - // Exploit the fact that the returned iterator (if any) is *not* bound to the lifetime - // of the chunk (it has an internal Arc). - .map(move |chunk| (Arc::clone(&chunk).iter_indices_owned(&timeline), chunk)) - .flat_map(move |(indices_iter, chunk)| { - map_chunk_indices_to_key_value_iter( - indices_iter, - chunk, - entity_path.clone(), - resolved_time_range, - ) - }) - } - - // all the rows! - let rows_to_chunk = query - .iter_all_data_results() - .filter(|data_result| data_result.is_visible(ctx)) - .flat_map(|data_result| { - entity_to_key_value_iter( - ctx, - &data_result.entity_path, - *timeline, - resolved_time_range, - ) - }) - .collect::>(); - - // - // Row sorting based on view properties. - // - - let mut rows = rows_to_chunk.keys().collect::>(); - - // apply sort key - match sort_key { - SortKey::Entity => {} // already correctly sorted - SortKey::Time => rows.sort_by_key(|(entity_path, time, _)| (*time, entity_path)), - }; - if sort_order == SortOrder::Descending { - rows.reverse(); - } - - // - // Scroll to focused item. - // - - let index_for_instance_path = |instance_path: &InstancePath| { - rows.iter() - .position(|(entity_path, _, _)| entity_path == &instance_path.entity_path) - }; - - let scroll_to_row = (sort_key == SortKey::Entity) - .then(|| { - ctx.focused_item.as_ref().and_then(|item| match item { - Item::AppId(_) - | Item::DataSource(_) - | Item::StoreId(_) - | Item::ComponentPath(_) - | Item::SpaceView(_) - | Item::Container(_) => None, - - // TODO(#6906): we shouldn't scroll when we originate the focus event. - // Note: we can't filter on space view id, because we really want to handle focus - // event from this view's blueprint tree. - Item::DataResult(_, instance_path) | Item::InstancePath(instance_path) => { - index_for_instance_path(instance_path) - } - }) - }) - .flatten(); - - // - // Drawing code. - // - - let entity_header = |ui: &mut egui::Ui| { - ui.strong("Entity"); - }; - let time_header = |ui: &mut egui::Ui| { - ui.strong("Time"); - }; - - // Draw the header row. - let header_ui = |mut row: egui_extras::TableRow<'_, '_>| { - match sort_key { - SortKey::Entity => { - row.col(entity_header); - row.col(time_header); - } - SortKey::Time => { - row.col(time_header); - row.col(entity_header); - } - } - - row.col(|ui| { - ui.strong("Row ID"); - }); - - for comp in &sorted_components { - row.col(|ui| { - ui.strong(comp.short_name()); - }); - } - }; - - let entity_ui = - |ui: &mut egui::Ui, entity_path: &EntityPath, latest_at_query: &LatestAtQuery| { - entity_path_button( - ctx, - latest_at_query, - ctx.recording(), - ui, - Some(query.space_view_id), - entity_path, - ); - }; - - let time_ui = |ui: &mut egui::Ui, time: &TimeInt| { - if ui - .button( - query - .timeline - .typ() - .format(*time, ctx.app_options.time_zone), - ) - .clicked() - { - ctx.rec_cfg.time_ctrl.write().set_time(*time); - } - }; - - // Draw a single line of the table. This is called for each _visible_ row. - let row_ui = |mut row: egui_extras::TableRow<'_, '_>| { - let row_key = rows[row.index()]; - let row_chunk = &rows_to_chunk[row_key]; - let (entity_path, time, row_id) = row_key; - - // Latest-at query corresponding to the current time - let latest_at_query = LatestAtQuery::new(*timeline, *time); - - match sort_key { - SortKey::Entity => { - row.col(|ui| entity_ui(ui, entity_path, &latest_at_query)); - row.col(|ui| time_ui(ui, time)); - } - SortKey::Time => { - row.col(|ui| time_ui(ui, time)); - row.col(|ui| entity_ui(ui, entity_path, &latest_at_query)); - } - }; - - row.col(|ui| { - row_id_ui(ui, row_id); - }); - - for component_name in &sorted_components { - row.col(|ui| { - let content = row_chunk.cell(*row_id, component_name); - - if let Some(content) = content { - ctx.component_ui_registry.ui_raw( - ctx, - ui, - UiLayout::List, - &latest_at_query, - ctx.recording(), - entity_path, - *component_name, - Some(*row_id), - &*content, - ); - } else { - ui.weak("-"); - } - }); - } - }; - - table_ui( - ui, - &sorted_components, - 3, // time, entity, row id - header_ui, - rows.len(), - row_ui, - scroll_to_row, - ); -} diff --git a/crates/viewer/re_space_view_dataframe/src/utils.rs b/crates/viewer/re_space_view_dataframe/src/utils.rs deleted file mode 100644 index b5b2cc938ee9..000000000000 --- a/crates/viewer/re_space_view_dataframe/src/utils.rs +++ /dev/null @@ -1,58 +0,0 @@ -use std::collections::BTreeSet; - -use re_chunk_store::{ChunkStore, LatestAtQuery}; -use re_entity_db::InstancePath; -use re_log_types::{EntityPath, Instance, Timeline}; -use re_viewer_context::{ViewQuery, ViewerContext}; - -/// Returns a sorted list of all entities that are visible in the view. -pub(crate) fn sorted_visible_entity_path( - ctx: &ViewerContext<'_>, - query: &ViewQuery<'_>, -) -> BTreeSet { - query - .iter_all_data_results() - .filter(|data_result| data_result.is_visible(ctx)) - .map(|data_result| &data_result.entity_path) - .cloned() - .collect() -} - -/// Returns a sorted, deduplicated iterator of all instance paths for a given entity. -pub(crate) fn sorted_instance_paths_for<'a>( - entity_path: &'a EntityPath, - store: &'a ChunkStore, - timeline: &Timeline, - latest_at_query: &'a LatestAtQuery, -) -> impl Iterator + 'a { - re_tracing::profile_function!(); - - // TODO(cmc): This should be using re_query. - - store - .all_components_on_timeline(timeline, entity_path) - .unwrap_or_default() - .into_iter() - .filter(|component_name| !component_name.is_indicator_component()) - .flat_map(|component_name| { - let num_instances = store - .latest_at_relevant_chunks(latest_at_query, entity_path, component_name) - .into_iter() - .filter_map(|chunk| { - let (index, unit) = chunk - .latest_at(latest_at_query, component_name) - .into_unit() - .and_then(|unit| unit.index(timeline).map(|index| (index, unit)))?; - - unit.component_batch_raw(&component_name) - .map(|array| (index, array)) - }) - .max_by_key(|(index, _array)| *index) - .map_or(0, |(_index, array)| array.len()); - - (0..num_instances).map(|i| Instance::from(i as u64)) - }) - .collect::>() // dedup and sort - .into_iter() - .map(|instance| InstancePath::instance(entity_path.clone(), instance)) -} diff --git a/crates/viewer/re_viewer/src/app.rs b/crates/viewer/re_viewer/src/app.rs index 053da21de0a1..7cd35e91dd5b 100644 --- a/crates/viewer/re_viewer/src/app.rs +++ b/crates/viewer/re_viewer/src/app.rs @@ -1240,10 +1240,11 @@ impl App { store_hub.clear_all_cloned_blueprints(); - // Reset egui, but keep the style: - let style = egui_ctx.style(); + // Reset egui: egui_ctx.memory_mut(|mem| *mem = Default::default()); - egui_ctx.set_style((*style).clone()); + + // Restore style: + re_ui::apply_style_and_install_loaders(egui_ctx); if let Err(err) = crate::reset_viewer_persistence() { re_log::warn!("Failed to reset viewer: {err}"); diff --git a/crates/viewer/re_viewer/src/web.rs b/crates/viewer/re_viewer/src/web.rs index 81b3aa125153..f3c29e2cae00 100644 --- a/crates/viewer/re_viewer/src/web.rs +++ b/crates/viewer/re_viewer/src/web.rs @@ -73,6 +73,7 @@ impl WebHandle { wgpu_options: crate::wgpu_options(app_options.render_backend.clone()), depth_buffer: 0, dithering: true, + ..Default::default() }; self.runner diff --git a/crates/viewer/re_viewer_context/src/component_ui_registry.rs b/crates/viewer/re_viewer_context/src/component_ui_registry.rs index f28dc537d92e..f94b96969378 100644 --- a/crates/viewer/re_viewer_context/src/component_ui_registry.rs +++ b/crates/viewer/re_viewer_context/src/component_ui_registry.rs @@ -96,10 +96,20 @@ impl UiLayout { pub fn label(self, ui: &mut egui::Ui, text: impl Into) -> egui::Response { let mut label = egui::Label::new(text); - match self { - Self::List => label = label.truncate(), - Self::Tooltip | Self::SelectionPanelLimitHeight | Self::SelectionPanelFull => { - label = label.wrap(); + // Respect set wrap_mode if already set + if ui.style().wrap_mode.is_none() { + match self { + Self::List => { + if ui.is_sizing_pass() { + // grow parent if needed - that's the point of a sizing pass + label = label.extend(); + } else { + label = label.truncate(); + } + } + Self::Tooltip | Self::SelectionPanelLimitHeight | Self::SelectionPanelFull => { + label = label.wrap(); + } } } @@ -125,9 +135,14 @@ impl UiLayout { match self { Self::List => { - // Elide - layout_job.wrap.max_rows = 1; - layout_job.wrap.break_anywhere = true; + layout_job.wrap.max_rows = 1; // We must fit on one line + if ui.is_sizing_pass() { + // grow parent if needed - that's the point of a sizing pass + layout_job.wrap.max_width = f32::INFINITY; + } else { + // Truncate + layout_job.wrap.break_anywhere = true; + } } Self::Tooltip => { layout_job.wrap.max_rows = 3;