From 0e38f50c17e19b7364988edaf80295f2e88e49ad Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 11 Oct 2024 11:42:07 +0200 Subject: [PATCH 1/5] QueryExpression::filtered_index is now optional --- crates/store/re_chunk_store/src/dataframe.rs | 45 +- crates/store/re_dataframe/examples/query.rs | 21 +- crates/store/re_dataframe/src/query.rs | 477 +++++++++++------- .../src/dataframe_ui.rs | 33 +- .../src/space_view_class.rs | 2 +- rerun_py/src/dataframe.rs | 62 ++- 6 files changed, 377 insertions(+), 263 deletions(-) diff --git a/crates/store/re_chunk_store/src/dataframe.rs b/crates/store/re_chunk_store/src/dataframe.rs index 10433e05e1af..e61c69b648fd 100644 --- a/crates/store/re_chunk_store/src/dataframe.rs +++ b/crates/store/re_chunk_store/src/dataframe.rs @@ -574,8 +574,7 @@ pub type IndexRange = ResolvedTimeRange; /// ``` // // TODO(cmc): ideally we'd like this to be the same type as the one used in the blueprint, possibly? -// TODO(cmc): Get rid of all re_dataframe (as opposed to re_dataframe) stuff and rename this. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Default, Debug, Clone, PartialEq, Eq, Hash)] pub struct QueryExpression { /// The subset of the database that the query will run on: a set of [`EntityPath`]s and their /// associated [`ComponentName`]s. @@ -618,17 +617,20 @@ pub struct QueryExpression { /// Only rows where at least 1 column contains non-null data at that index will be kept in the /// final dataset. /// - /// Example: `Timeline("frame")`. + /// If left unspecified, the results will only contain static data. + /// + /// Examples: `Some(Timeline("frame"))`, `None` (only static data). // // TODO(cmc): this has to be a selector otherwise this is a horrible UX. - pub filtered_index: Timeline, + pub filtered_index: Option, /// The range of index values used to filter out _rows_ from the view contents. /// /// Only rows where at least 1 of the view-contents contains non-null data within that range will be kept in /// the final dataset. /// - /// This is ignored if [`QueryExpression::using_index_values`] is set. + /// * This has no effect if `filtered_index` isn't set. + /// * This has no effect if [`QueryExpression::using_index_values`] is set. /// /// Example: `ResolvedTimeRange(10, 20)`. pub filtered_index_range: Option, @@ -638,7 +640,9 @@ pub struct QueryExpression { /// Only rows where at least 1 column contains non-null data at these specific values will be kept /// in the final dataset. /// - /// This is ignored if [`QueryExpression::using_index_values`] is set. + /// * This has no effect if `filtered_index` isn't set. + /// * This has no effect if [`QueryExpression::using_index_values`] is set. + /// * Using [`TimeInt::STATIC`] as index value has no effect. /// /// Example: `[TimeInt(12), TimeInt(14)]`. pub filtered_index_values: Option>, @@ -650,10 +654,10 @@ pub struct QueryExpression { /// The semantics of the query are consistent with all other settings: the results will be /// sorted on the `filtered_index`, and only contain unique index values. /// - /// The order of the samples will be respected in the final result. - /// - /// If [`QueryExpression::using_index_values`] is set, it overrides both [`QueryExpression::filtered_index_range`] - /// and [`QueryExpression::filtered_index_values`]. + /// * This has no effect if `filtered_index` isn't set. + /// * If set, this overrides both [`QueryExpression::filtered_index_range`] and + /// [`QueryExpression::filtered_index_values`]. + /// * Using [`TimeInt::STATIC`] as index value has no effect. /// /// Example: `[TimeInt(12), TimeInt(14)]`. pub using_index_values: Option>, @@ -684,27 +688,6 @@ pub struct QueryExpression { pub selection: Option>, } -impl QueryExpression { - #[inline] - pub fn new(index: impl Into) -> Self { - let index = index.into(); - - Self { - view_contents: None, - include_semantically_empty_columns: false, - include_indicator_columns: false, - include_tombstone_columns: false, - filtered_index: index, - filtered_index_range: None, - filtered_index_values: None, - using_index_values: None, - filtered_point_of_view: None, - sparse_fill_strategy: SparseFillStrategy::None, - selection: None, - } - } -} - // --- impl ChunkStore { diff --git a/crates/store/re_dataframe/examples/query.rs b/crates/store/re_dataframe/examples/query.rs index 64d2f302455c..7d374587d45b 100644 --- a/crates/store/re_dataframe/examples/query.rs +++ b/crates/store/re_dataframe/examples/query.rs @@ -59,15 +59,18 @@ fn main() -> anyhow::Result<()> { cache: &query_cache, }; - let mut query = QueryExpression::new(timeline); - query.view_contents = Some( - query_engine - .iter_entity_paths(&entity_path_filter) - .map(|entity_path| (entity_path, None)) - .collect(), - ); - query.filtered_index_range = Some(ResolvedTimeRange::new(time_from, time_to)); - query.sparse_fill_strategy = SparseFillStrategy::LatestAtGlobal; + let query = QueryExpression { + filtered_index: Some(timeline), + view_contents: Some( + query_engine + .iter_entity_paths(&entity_path_filter) + .map(|entity_path| (entity_path, None)) + .collect(), + ), + filtered_index_range: Some(ResolvedTimeRange::new(time_from, time_to)), + sparse_fill_strategy: SparseFillStrategy::LatestAtGlobal, + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); diff --git a/crates/store/re_dataframe/src/query.rs b/crates/store/re_dataframe/src/query.rs index 2eb70680b956..6a252dd1f097 100644 --- a/crates/store/re_dataframe/src/query.rs +++ b/crates/store/re_dataframe/src/query.rs @@ -22,7 +22,7 @@ use re_chunk::{ Chunk, ComponentName, EntityPath, RangeQuery, RowId, TimeInt, Timeline, UnitChunkShared, }; use re_chunk_store::{ - ColumnDescriptor, ColumnSelector, ComponentColumnDescriptor, ComponentColumnSelector, + ColumnDescriptor, ColumnSelector, ComponentColumnDescriptor, ComponentColumnSelector, Index, IndexValue, JoinEncoding, QueryExpression, SparseFillStrategy, TimeColumnDescriptor, TimeColumnSelector, }; @@ -33,6 +33,17 @@ use crate::{QueryEngine, RecordBatch}; // --- +// TODO: +// +// * Default mode: +// * filtered_index = Some(something) +// * TimeInt::STATIC is never ever returned by default +// * Static values are repeated every row and override everything +// +// * Static mode: +// * filtered_index = None +// * TimeInt::STATIC is the only index value ever returned + // TODO(cmc): (no specific order) (should we make issues for these?) // * [x] basic thing working // * [x] custom selection @@ -87,6 +98,12 @@ struct QueryHandleState { /// See also [`QueryHandleState::arrow_schema`]. selected_contents: Vec<(usize, ColumnDescriptor)>, + /// The actual index filter in use, since the user-specified one is optional. + /// + /// This just defaults to `Index::default()` if ther user hasn't specified any: the actual + /// value is irrelevant since this means we are only concerned with static data anyway. + filtered_index: Index, + /// The Arrow schema that corresponds to the `selected_contents`. /// /// All returned rows will have this schema. @@ -150,6 +167,9 @@ impl QueryHandle<'_> { fn init_(&self) -> QueryHandleState { re_tracing::profile_scope!("init"); + // The timeline doesn't matter if we're running in static-only mode. + let filtered_index = self.query.filtered_index.unwrap_or_default(); + // 1. Compute the schema for the query. let view_contents = self.engine.store.schema_for_query(&self.query); @@ -191,7 +211,7 @@ impl QueryHandle<'_> { .unwrap_or(ResolvedTimeRange::EVERYTHING) }; - RangeQuery::new(self.query.filtered_index, index_range) + RangeQuery::new(filtered_index, index_range) .keep_extra_timelines(true) // we want all the timelines we can get! .keep_extra_components(false) }; @@ -248,7 +268,7 @@ impl QueryHandle<'_> { // the time range for the specific component of interest. chunk .timelines() - .get(&self.query.filtered_index) + .get(&filtered_index) .map(|time_column| time_column.time_range()) .map_or(TimeInt::STATIC, |time_range| time_range.min()) }); @@ -280,7 +300,7 @@ impl QueryHandle<'_> { } else { chunk .timelines() - .get(&self.query.filtered_index) + .get(&filtered_index) .map(|time_column| Either::Right(time_column.times())) } }) @@ -298,6 +318,7 @@ impl QueryHandle<'_> { QueryHandleState { view_contents, selected_contents, + filtered_index, arrow_schema, view_chunks, cur_row: AtomicU64::new(0), @@ -574,7 +595,7 @@ impl QueryHandle<'_> { "the query cache should have already taken care of sorting (and densifying!) the chunk", ); - let chunk = chunk.deduped_latest_on_index(&self.query.filtered_index); + let chunk = chunk.deduped_latest_on_index(&query.timeline); (AtomicU64::default(), chunk) }) @@ -677,7 +698,7 @@ impl QueryHandle<'_> { for (cursor, chunk) in chunks { // NOTE: The chunk has been densified already: its global time range is the same as // the time range for the specific component of interest. - let Some(time_column) = chunk.timelines().get(&self.query.filtered_index) else { + let Some(time_column) = chunk.timelines().get(&state.filtered_index) else { continue; }; @@ -800,9 +821,7 @@ impl QueryHandle<'_> { // TODO(cmc): make this a tiny bit smarter so we can remove the need for // deduped_latest_on_index, which would be very welcome right now given we don't // have an Arrow ListView at our disposal. - let cur_indices = cur_chunk - .iter_indices(&self.query.filtered_index) - .collect_vec(); + let cur_indices = cur_chunk.iter_indices(&state.filtered_index).collect_vec(); let (index_value, cur_row_id) = 'walk: loop { let Some((index_value, cur_row_id)) = cur_indices.get(cur_cursor_value as usize).copied() @@ -879,7 +898,7 @@ impl QueryHandle<'_> { // consecutive nulls etc). Later. let query = - re_chunk::LatestAtQuery::new(self.query.filtered_index, *cur_index_value); + re_chunk::LatestAtQuery::new(state.filtered_index, *cur_index_value); let results = self.engine.cache.latest_at( self.engine.store, @@ -959,11 +978,11 @@ impl QueryHandle<'_> { // The current index value (if temporal) should be the one returned for the // queried index, no matter what. max_value_per_index.insert( - self.query.filtered_index, + state.filtered_index, ( *cur_index_value, ArrowPrimitiveArray::::from_vec(vec![cur_index_value.as_i64()]) - .to(self.query.filtered_index.datatype()) + .to(state.filtered_index.datatype()) .to_boxed(), ), ); @@ -1144,8 +1163,11 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); - let query = QueryExpression::new(timeline); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); + let query = QueryExpression { + filtered_index, + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1189,9 +1211,12 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); - let mut query = QueryExpression::new(timeline); - query.sparse_fill_strategy = SparseFillStrategy::LatestAtGlobal; + let filtered_index = Some(Timeline::new_sequence("frame_nr")); + let query = QueryExpression { + filtered_index, + sparse_fill_strategy: SparseFillStrategy::LatestAtGlobal, + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1235,9 +1260,12 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); - let mut query = QueryExpression::new(timeline); - query.filtered_index_range = Some(ResolvedTimeRange::new(30, 60)); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); + let query = QueryExpression { + filtered_index, + filtered_index_range: Some(ResolvedTimeRange::new(30, 60)), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1281,15 +1309,18 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); - let mut query = QueryExpression::new(timeline); - query.filtered_index_values = Some( - [0, 30, 60, 90] - .into_iter() - .map(TimeInt::new_temporal) - .chain(std::iter::once(TimeInt::STATIC)) - .collect(), - ); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); + let query = QueryExpression { + filtered_index, + filtered_index_values: Some( + [0, 30, 60, 90] + .into_iter() + .map(TimeInt::new_temporal) + .chain(std::iter::once(TimeInt::STATIC)) + .collect(), + ), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1333,18 +1364,21 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); // vanilla { - let mut query = QueryExpression::new(timeline); - query.using_index_values = Some( - [0, 15, 30, 30, 45, 60, 75, 90] - .into_iter() - .map(TimeInt::new_temporal) - .chain(std::iter::once(TimeInt::STATIC)) - .collect(), - ); + let query = QueryExpression { + filtered_index, + using_index_values: Some( + [0, 15, 30, 30, 45, 60, 75, 90] + .into_iter() + .map(TimeInt::new_temporal) + .chain(std::iter::once(TimeInt::STATIC)) + .collect(), + ), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1375,16 +1409,19 @@ mod tests { } // sparse-filled - if true { - let mut query = QueryExpression::new(timeline); - query.using_index_values = Some( - [0, 15, 30, 30, 45, 60, 75, 90] - .into_iter() - .map(TimeInt::new_temporal) - .chain(std::iter::once(TimeInt::STATIC)) - .collect(), - ); - query.sparse_fill_strategy = SparseFillStrategy::LatestAtGlobal; + { + let query = QueryExpression { + filtered_index, + using_index_values: Some( + [0, 15, 30, 30, 45, 60, 75, 90] + .into_iter() + .map(TimeInt::new_temporal) + .chain(std::iter::once(TimeInt::STATIC)) + .collect(), + ), + sparse_fill_strategy: SparseFillStrategy::LatestAtGlobal, + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1429,17 +1466,20 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); let entity_path: EntityPath = "this/that".into(); // non-existing entity { - let mut query = QueryExpression::new(timeline); - query.filtered_point_of_view = Some(ComponentColumnSelector { - entity_path: "no/such/entity".into(), - component: MyPoint::name(), - join_encoding: Default::default(), - }); + let query = QueryExpression { + filtered_index, + filtered_point_of_view: Some(ComponentColumnSelector { + entity_path: "no/such/entity".into(), + component: MyPoint::name(), + join_encoding: Default::default(), + }), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1461,12 +1501,15 @@ mod tests { // non-existing component { - let mut query = QueryExpression::new(timeline); - query.filtered_point_of_view = Some(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: "AComponentColumnThatDoesntExist".into(), - join_encoding: Default::default(), - }); + let query = QueryExpression { + filtered_index, + filtered_point_of_view: Some(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: "AComponentColumnThatDoesntExist".into(), + join_encoding: Default::default(), + }), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1488,12 +1531,15 @@ mod tests { // MyPoint { - let mut query = QueryExpression::new(timeline); - query.filtered_point_of_view = Some(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyPoint::name(), - join_encoding: Default::default(), - }); + let query = QueryExpression { + filtered_index, + filtered_point_of_view: Some(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyPoint::name(), + join_encoding: Default::default(), + }), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1525,12 +1571,15 @@ mod tests { // MyColor { - let mut query = QueryExpression::new(timeline); - query.filtered_point_of_view = Some(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyColor::name(), - join_encoding: Default::default(), - }); + let query = QueryExpression { + filtered_index, + filtered_point_of_view: Some(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyColor::name(), + join_encoding: Default::default(), + }), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1576,16 +1625,19 @@ mod tests { }; let entity_path: EntityPath = "this/that".into(); - let timeline = Timeline::new_sequence("frame_nr"); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); // empty view { - let mut query = QueryExpression::new(timeline); - query.view_contents = Some( - [(entity_path.clone(), Some(Default::default()))] - .into_iter() - .collect(), - ); + let query = QueryExpression { + filtered_index, + view_contents: Some( + [(entity_path.clone(), Some(Default::default()))] + .into_iter() + .collect(), + ), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1606,23 +1658,26 @@ mod tests { } { - let mut query = QueryExpression::new(timeline); - query.view_contents = Some( - [( - entity_path.clone(), - Some( - [ - MyLabel::name(), - MyColor::name(), - "AColumnThatDoesntEvenExist".into(), - ] - .into_iter() - .collect(), - ), - )] - .into_iter() - .collect(), - ); + let query = QueryExpression { + filtered_index, + view_contents: Some( + [( + entity_path.clone(), + Some( + [ + MyLabel::name(), + MyColor::name(), + "AColumnThatDoesntEvenExist".into(), + ] + .into_iter() + .collect(), + ), + )] + .into_iter() + .collect(), + ), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1667,12 +1722,15 @@ mod tests { }; let entity_path: EntityPath = "this/that".into(); - let timeline = Timeline::new_sequence("frame_nr"); + let filtered_index = Timeline::new_sequence("frame_nr"); // empty selection { - let mut query = QueryExpression::new(timeline); - query.selection = Some(vec![]); + let query = QueryExpression { + filtered_index: Some(filtered_index), + selection: Some(vec![]), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1694,18 +1752,21 @@ mod tests { // only indices (+ duplication) { - let mut query = QueryExpression::new(timeline); - query.selection = Some(vec![ - ColumnSelector::Time(TimeColumnSelector { - timeline: *timeline.name(), - }), - ColumnSelector::Time(TimeColumnSelector { - timeline: *timeline.name(), - }), - ColumnSelector::Time(TimeColumnSelector { - timeline: "ATimeColumnThatDoesntExist".into(), - }), - ]); + let query = QueryExpression { + filtered_index: Some(filtered_index), + selection: Some(vec![ + ColumnSelector::Time(TimeColumnSelector { + timeline: *filtered_index.name(), + }), + ColumnSelector::Time(TimeColumnSelector { + timeline: *filtered_index.name(), + }), + ColumnSelector::Time(TimeColumnSelector { + timeline: "ATimeColumnThatDoesntExist".into(), + }), + ]), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1735,29 +1796,32 @@ mod tests { // only components (+ duplication) { - let mut query = QueryExpression::new(timeline); - query.selection = Some(vec![ - ColumnSelector::Component(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyColor::name(), - join_encoding: Default::default(), - }), - ColumnSelector::Component(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyColor::name(), - join_encoding: Default::default(), - }), - ColumnSelector::Component(ComponentColumnSelector { - entity_path: "non_existing_entity".into(), - component: MyColor::name(), - join_encoding: Default::default(), - }), - ColumnSelector::Component(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: "AComponentColumnThatDoesntExist".into(), - join_encoding: Default::default(), - }), - ]); + let query = QueryExpression { + filtered_index: Some(filtered_index), + selection: Some(vec![ + ColumnSelector::Component(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyColor::name(), + join_encoding: Default::default(), + }), + ColumnSelector::Component(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyColor::name(), + join_encoding: Default::default(), + }), + ColumnSelector::Component(ComponentColumnSelector { + entity_path: "non_existing_entity".into(), + component: MyColor::name(), + join_encoding: Default::default(), + }), + ColumnSelector::Component(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: "AComponentColumnThatDoesntExist".into(), + join_encoding: Default::default(), + }), + ]), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1802,46 +1866,49 @@ mod tests { }; let entity_path: EntityPath = "this/that".into(); - let timeline = Timeline::new_sequence("frame_nr"); + let filtered_index = Timeline::new_sequence("frame_nr"); // only components { - let mut query = QueryExpression::new(timeline); - query.view_contents = Some( - [( - entity_path.clone(), - Some([MyColor::name(), MyLabel::name()].into_iter().collect()), - )] - .into_iter() - .collect(), - ); - query.selection = Some(vec![ - ColumnSelector::Time(TimeColumnSelector { - timeline: *timeline.name(), - }), - ColumnSelector::Time(TimeColumnSelector { - timeline: *Timeline::log_time().name(), - }), - ColumnSelector::Time(TimeColumnSelector { - timeline: *Timeline::log_tick().name(), - }), - // - ColumnSelector::Component(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyPoint::name(), - join_encoding: Default::default(), - }), - ColumnSelector::Component(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyColor::name(), - join_encoding: Default::default(), - }), - ColumnSelector::Component(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyLabel::name(), - join_encoding: Default::default(), - }), - ]); + let query = QueryExpression { + filtered_index: Some(filtered_index), + view_contents: Some( + [( + entity_path.clone(), + Some([MyColor::name(), MyLabel::name()].into_iter().collect()), + )] + .into_iter() + .collect(), + ), + selection: Some(vec![ + ColumnSelector::Time(TimeColumnSelector { + timeline: *filtered_index.name(), + }), + ColumnSelector::Time(TimeColumnSelector { + timeline: *Timeline::log_time().name(), + }), + ColumnSelector::Time(TimeColumnSelector { + timeline: *Timeline::log_tick().name(), + }), + // + ColumnSelector::Component(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyPoint::name(), + join_encoding: Default::default(), + }), + ColumnSelector::Component(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyColor::name(), + join_encoding: Default::default(), + }), + ColumnSelector::Component(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyLabel::name(), + join_encoding: Default::default(), + }), + ]), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1889,13 +1956,16 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); let entity_path = EntityPath::from("this/that"); // barebones { - let mut query = QueryExpression::new(timeline); - query.view_contents = Some([(entity_path.clone(), None)].into_iter().collect()); + let query = QueryExpression { + filtered_index, + view_contents: Some([(entity_path.clone(), None)].into_iter().collect()), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1927,9 +1997,12 @@ mod tests { // sparse-filled { - let mut query = QueryExpression::new(timeline); - query.view_contents = Some([(entity_path.clone(), None)].into_iter().collect()); - query.sparse_fill_strategy = SparseFillStrategy::LatestAtGlobal; + let query = QueryExpression { + filtered_index, + view_contents: Some([(entity_path.clone(), None)].into_iter().collect()), + sparse_fill_strategy: SparseFillStrategy::LatestAtGlobal, + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -1978,12 +2051,15 @@ mod tests { cache: &query_cache, }; - let timeline = Timeline::new_sequence("frame_nr"); + let filtered_index = Some(Timeline::new_sequence("frame_nr")); let entity_path = EntityPath::from("this/that"); // basic { - let query = QueryExpression::new(timeline); + let query = QueryExpression { + filtered_index, + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -2017,12 +2093,15 @@ mod tests { // with pov { - let mut query = QueryExpression::new(timeline); - query.filtered_point_of_view = Some(ComponentColumnSelector { - entity_path: entity_path.clone(), - component: MyPoint::name(), - join_encoding: Default::default(), - }); + let query = QueryExpression { + filtered_index, + filtered_point_of_view: Some(ComponentColumnSelector { + entity_path: entity_path.clone(), + component: MyPoint::name(), + join_encoding: Default::default(), + }), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -2056,14 +2135,17 @@ mod tests { // with sampling { - let mut query = QueryExpression::new(timeline); - query.using_index_values = Some( - [0, 15, 30, 30, 45, 60, 75, 90] - .into_iter() - .map(TimeInt::new_temporal) - .chain(std::iter::once(TimeInt::STATIC)) - .collect(), - ); + let query = QueryExpression { + filtered_index, + using_index_values: Some( + [0, 15, 30, 30, 45, 60, 75, 90] + .into_iter() + .map(TimeInt::new_temporal) + .chain(std::iter::once(TimeInt::STATIC)) + .collect(), + ), + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); @@ -2097,8 +2179,11 @@ mod tests { // with sparse-fill { - let mut query = QueryExpression::new(timeline); - query.sparse_fill_strategy = SparseFillStrategy::LatestAtGlobal; + let query = QueryExpression { + filtered_index, + sparse_fill_strategy: SparseFillStrategy::LatestAtGlobal, + ..Default::default() + }; eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone()); diff --git a/crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs b/crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs index 0fb0cba0ff94..60c8fb775938 100644 --- a/crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs +++ b/crates/viewer/re_space_view_dataframe/src/dataframe_ui.rs @@ -200,7 +200,8 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> { fn prepare(&mut self, info: &egui_table::PrefetchInfo) { re_tracing::profile_function!(); - let timeline = self.query_handle.query().filtered_index; + // TODO(ab): actual static-only support + let filtered_index = self.query_handle.query().filtered_index.unwrap_or_default(); self.query_handle .seek_to_row(info.visible_rows.start as usize); @@ -208,8 +209,12 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> { .take((info.visible_rows.end - info.visible_rows.start) as usize) .collect(); - let data = - RowsDisplayData::try_new(&info.visible_rows, data, self.selected_columns, &timeline); + let data = RowsDisplayData::try_new( + &info.visible_rows, + data, + self.selected_columns, + &filtered_index, + ); self.display_data = data.context("Failed to create display data"); } @@ -253,13 +258,19 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> { } else if cell.row_nr == 1 { let column = &self.selected_columns[cell.col_range.start]; + // TODO(ab): actual static-only support + let filtered_index = + self.query_handle.query().filtered_index.unwrap_or_default(); + // if this column can actually be hidden, then that's the corresponding action let hide_action = match column { - ColumnDescriptor::Time(desc) => (desc.timeline - != self.query_handle.query().filtered_index) - .then(|| HideColumnAction::HideTimeColumn { - timeline_name: *desc.timeline.name(), - }), + ColumnDescriptor::Time(desc) => { + (desc.timeline != filtered_index).then(|| { + HideColumnAction::HideTimeColumn { + timeline_name: *desc.timeline.name(), + } + }) + } ColumnDescriptor::Component(desc) => { Some(HideColumnAction::HideComponentColumn { @@ -331,8 +342,10 @@ impl<'a> egui_table::TableDelegate for DataframeTableDelegate<'a> { .try_decode_time(batch_row_idx) }) .unwrap_or(TimeInt::MAX); - let latest_at_query = - LatestAtQuery::new(self.query_handle.query().filtered_index, timestamp); + + // TODO(ab): actual static-only support + let filtered_index = self.query_handle.query().filtered_index.unwrap_or_default(); + let latest_at_query = LatestAtQuery::new(filtered_index, timestamp); if ui.is_sizing_pass() { ui.style_mut().wrap_mode = Some(egui::TextWrapMode::Extend); diff --git a/crates/viewer/re_space_view_dataframe/src/space_view_class.rs b/crates/viewer/re_space_view_dataframe/src/space_view_class.rs index 98178d38a619..ec1c2b934de4 100644 --- a/crates/viewer/re_space_view_dataframe/src/space_view_class.rs +++ b/crates/viewer/re_space_view_dataframe/src/space_view_class.rs @@ -145,7 +145,7 @@ mode sets the default time range to _everything_. You can override this in the s let mut dataframe_query = re_chunk_store::QueryExpression { view_contents: Some(view_contents), - filtered_index: view_query.timeline(ctx)?, + filtered_index: Some(view_query.timeline(ctx)?), filtered_index_range: Some(view_query.filter_by_range()?), filtered_point_of_view: view_query.filter_by_event()?, sparse_fill_strategy, diff --git a/rerun_py/src/dataframe.rs b/rerun_py/src/dataframe.rs index 6651fd2020b6..97b6f6bd5d47 100644 --- a/rerun_py/src/dataframe.rs +++ b/rerun_py/src/dataframe.rs @@ -458,11 +458,21 @@ impl PyRecordingView { } fn filter_range_sequence(&self, start: i64, end: i64) -> PyResult { - if self.query_expression.filtered_index.typ() != TimeType::Sequence { - return Err(PyValueError::new_err(format!( - "Index for {} is not a sequence.", - self.query_expression.filtered_index.name() - ))); + match self.query_expression.filtered_index.as_ref() { + Some(filtered_index) if filtered_index.typ() != TimeType::Sequence => { + return Err(PyValueError::new_err(format!( + "Index for {} is not a sequence.", + filtered_index.name() + ))); + } + + Some(_) => {} + + None => { + return Err(PyValueError::new_err( + "Specify an index to filter on first.".to_owned(), + )); + } } let start = if let Ok(seq) = re_chunk::TimeInt::try_from(start) { @@ -499,11 +509,21 @@ impl PyRecordingView { } fn filter_range_seconds(&self, start: f64, end: f64) -> PyResult { - if self.query_expression.filtered_index.typ() != TimeType::Time { - return Err(PyValueError::new_err(format!( - "Index for {} is not temporal.", - self.query_expression.filtered_index.name() - ))); + match self.query_expression.filtered_index.as_ref() { + Some(filtered_index) if filtered_index.typ() != TimeType::Time => { + return Err(PyValueError::new_err(format!( + "Index for {} is not temporal.", + filtered_index.name() + ))); + } + + Some(_) => {} + + None => { + return Err(PyValueError::new_err( + "Specify an index to filter on first.".to_owned(), + )); + } } let start = re_sdk::Time::from_seconds_since_epoch(start); @@ -521,11 +541,21 @@ impl PyRecordingView { } fn filter_range_nanos(&self, start: i64, end: i64) -> PyResult { - if self.query_expression.filtered_index.typ() != TimeType::Time { - return Err(PyValueError::new_err(format!( - "Index for {} is not temporal.", - self.query_expression.filtered_index.name() - ))); + match self.query_expression.filtered_index.as_ref() { + Some(filtered_index) if filtered_index.typ() != TimeType::Time => { + return Err(PyValueError::new_err(format!( + "Index for {} is not temporal.", + filtered_index.name() + ))); + } + + Some(_) => {} + + None => { + return Err(PyValueError::new_err( + "Specify an index to filter on first.".to_owned(), + )); + } } let start = re_sdk::Time::from_ns_since_epoch(start); @@ -673,7 +703,7 @@ impl PyRecording { include_semantically_empty_columns: false, include_indicator_columns: false, include_tombstone_columns: false, - filtered_index: timeline.timeline, + filtered_index: Some(timeline.timeline), filtered_index_range: None, filtered_index_values: None, using_index_values: None, From 981fe473b3eee507c39b2aacf3031ee8eddbd31b Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 11 Oct 2024 16:35:50 +0200 Subject: [PATCH 2/5] update test suite for new static semantics --- crates/store/re_dataframe/src/query.rs | 202 +++++++++++++++---------- 1 file changed, 120 insertions(+), 82 deletions(-) diff --git a/crates/store/re_dataframe/src/query.rs b/crates/store/re_dataframe/src/query.rs index 6a252dd1f097..584982f64550 100644 --- a/crates/store/re_dataframe/src/query.rs +++ b/crates/store/re_dataframe/src/query.rs @@ -1164,37 +1164,75 @@ mod tests { }; let filtered_index = Some(Timeline::new_sequence("frame_nr")); - let query = QueryExpression { - filtered_index, - ..Default::default() - }; - eprintln!("{query:#?}:"); - let query_handle = query_engine.query(query.clone()); - assert_eq!( - query_engine.query(query.clone()).into_iter().count() as u64, - query_handle.num_rows() - ); - let dataframe = concatenate_record_batches( - query_handle.schema().clone(), - &query_handle.into_batch_iter().collect_vec(), - ); - eprintln!("{dataframe}"); + // static + { + let query = QueryExpression { + ..Default::default() + }; + eprintln!("{query:#?}:"); - let got = format!("{:#?}", dataframe.data.iter().collect_vec()); - let expected = unindent::unindent( - "\ - [ - Int64[None, 10, 20, 30, 40, 50, 60, 70], - Timestamp(Nanosecond, None)[None, 1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, None, 1970-01-01 00:00:00.000000070], - ListArray[None, None, None, [2], [3], [4], None, [6]], - ListArray[[c], None, None, None, None, None, None, None], - ListArray[None, [{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 5, y: 5}], [{x: 8, y: 8}]], - ]\ - " - ); + let query_handle = query_engine.query(query.clone()); + assert_eq!( + query_engine.query(query.clone()).into_iter().count() as u64, + query_handle.num_rows() + ); + let dataframe = concatenate_record_batches( + query_handle.schema().clone(), + &query_handle.into_batch_iter().collect_vec(), + ); + eprintln!("{dataframe}"); - similar_asserts::assert_eq!(expected, got); + let got = format!("{:#?}", dataframe.data.iter().collect_vec()); + let expected = unindent::unindent( + "\ + [ + Int64[None], + Timestamp(Nanosecond, None)[None], + ListArray[None], + ListArray[[c]], + ListArray[None], + ]\ + ", + ); + + similar_asserts::assert_eq!(expected, got); + } + + // temporal + { + let query = QueryExpression { + filtered_index, + ..Default::default() + }; + eprintln!("{query:#?}:"); + + let query_handle = query_engine.query(query.clone()); + assert_eq!( + query_engine.query(query.clone()).into_iter().count() as u64, + query_handle.num_rows() + ); + let dataframe = concatenate_record_batches( + query_handle.schema().clone(), + &query_handle.into_batch_iter().collect_vec(), + ); + eprintln!("{dataframe}"); + + let got = format!("{:#?}", dataframe.data.iter().collect_vec()); + let expected = unindent::unindent( + "\ + [ + Int64[10, 20, 30, 40, 50, 60, 70], + Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, None, 1970-01-01 00:00:00.000000070], + ListArray[None, None, [2], [3], [4], None, [6]], + ListArray[[c], [c], [c], [c], [c], [c], [c]], + ListArray[[{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 5, y: 5}], [{x: 8, y: 8}]], + ]\ + " + ); + + similar_asserts::assert_eq!(expected, got); + } Ok(()) } @@ -1234,11 +1272,11 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 10, 20, 30, 40, 50, 60, 70], - Timestamp(Nanosecond, None)[None, 1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, None, 1970-01-01 00:00:00.000000070], - ListArray[None, None, None, [2], [3], [4], [4], [6]], - ListArray[[c], [c], [c], [c], [c], [c], [c], [c]], - ListArray[None, [{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 5, y: 5}], [{x: 8, y: 8}]], + Int64[10, 20, 30, 40, 50, 60, 70], + Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, None, 1970-01-01 00:00:00.000000070], + ListArray[None, None, [2], [3], [4], [4], [6]], + ListArray[[c], [c], [c], [c], [c], [c], [c]], + ListArray[[{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 5, y: 5}], [{x: 8, y: 8}]], ]\ " ); @@ -1283,11 +1321,11 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 30, 40, 50, 60], - Timestamp(Nanosecond, None)[None, None, None, 1970-01-01 00:00:00.000000050, None], - ListArray[None, [2], [3], [4], None], - ListArray[[c], None, None, None, None], - ListArray[None, [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 5, y: 5}]], + Int64[30, 40, 50, 60], + Timestamp(Nanosecond, None)[None, None, 1970-01-01 00:00:00.000000050, None], + ListArray[[2], [3], [4], None], + ListArray[[c], [c], [c], [c]], + ListArray[[{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 5, y: 5}]], ]\ ", ); @@ -1338,11 +1376,11 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 30, 60], - Timestamp(Nanosecond, None)[None, None, None], - ListArray[None, [2], None], - ListArray[[c], None, None], - ListArray[None, [{x: 2, y: 2}], [{x: 5, y: 5}]], + Int64[30, 60], + Timestamp(Nanosecond, None)[None, None], + ListArray[[2], None], + ListArray[[c], [c]], + ListArray[[{x: 2, y: 2}], [{x: 5, y: 5}]], ]\ ", ); @@ -1396,11 +1434,11 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 0, 15, 30, 45, 60, 75, 90], - Timestamp(Nanosecond, None)[None, None, None, None, None, None, None, None], - ListArray[None, None, None, [2], None, None, None, None], - ListArray[[c], None, None, None, None, None, None, None], - ListArray[None, None, None, [{x: 2, y: 2}], None, [{x: 5, y: 5}], None, None], + Int64[0, 15, 30, 45, 60, 75, 90], + Timestamp(Nanosecond, None)[None, None, None, None, None, None, None], + ListArray[None, None, [2], None, None, None, None], + ListArray[[c], [c], [c], [c], [c], [c], [c]], + ListArray[None, None, [{x: 2, y: 2}], None, [{x: 5, y: 5}], None, None], ]\ ", ); @@ -1439,11 +1477,11 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 0, 15, 30, 45, 60, 75, 90], - Timestamp(Nanosecond, None)[None, None, 1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000070, 1970-01-01 00:00:00.000000070], - ListArray[None, None, None, [2], [3], [4], [6], [6]], - ListArray[[c], [c], [c], [c], [c], [c], [c], [c]], - ListArray[None, None, [{x: 0, y: 0}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 5, y: 5}], [{x: 8, y: 8}], [{x: 8, y: 8}]], + Int64[0, 15, 30, 45, 60, 75, 90], + Timestamp(Nanosecond, None)[None, 1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000070, 1970-01-01 00:00:00.000000070], + ListArray[None, None, [2], [3], [4], [6], [6]], + ListArray[[c], [c], [c], [c], [c], [c], [c]], + ListArray[None, [{x: 0, y: 0}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 5, y: 5}], [{x: 8, y: 8}], [{x: 8, y: 8}]], ]\ ", ); @@ -1560,7 +1598,7 @@ mod tests { Int64[10, 20, 30, 40, 50, 60, 70], Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, None, 1970-01-01 00:00:00.000000070], ListArray[None, None, [2], [3], [4], None, [6]], - ListArray[None, None, None, None, None, None, None], + ListArray[[c], [c], [c], [c], [c], [c], [c]], ListArray[[{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 5, y: 5}], [{x: 8, y: 8}]], ]\ " @@ -1600,7 +1638,7 @@ mod tests { Int64[30, 40, 50, 70], Timestamp(Nanosecond, None)[None, None, 1970-01-01 00:00:00.000000050, 1970-01-01 00:00:00.000000070], ListArray[[2], [3], [4], [6]], - ListArray[None, None, None, None], + ListArray[[c], [c], [c], [c]], ListArray[[{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [{x: 8, y: 8}]], ]\ ", @@ -1695,10 +1733,10 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 30, 40, 50, 70], - Timestamp(Nanosecond, None)[None, None, None, None, None], - ListArray[None, [2], [3], [4], [6]], - ListArray[[c], None, None, None, None], + Int64[30, 40, 50, 70], + Timestamp(Nanosecond, None)[None, None, None, None], + ListArray[[2], [3], [4], [6]], + ListArray[[c], [c], [c], [c]], ]\ ", ); @@ -1784,9 +1822,9 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 10, 20, 30, 40, 50, 60, 70], - Int64[None, 10, 20, 30, 40, 50, 60, 70], - NullArray(8), + Int64[10, 20, 30, 40, 50, 60, 70], + Int64[10, 20, 30, 40, 50, 60, 70], + NullArray(7), ]\ ", ); @@ -1839,10 +1877,10 @@ mod tests { let expected = unindent::unindent( "\ [ - ListArray[None, None, None, [2], [3], [4], None, [6]], - ListArray[None, None, None, [2], [3], [4], None, [6]], - NullArray(8), - NullArray(8), + ListArray[None, None, [2], [3], [4], None, [6]], + ListArray[None, None, [2], [3], [4], None, [6]], + NullArray(7), + NullArray(7), ]\ ", ); @@ -1926,12 +1964,12 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 30, 40, 50, 70], - Timestamp(Nanosecond, None)[None, None, None, None, None], - NullArray(5), - NullArray(5), - ListArray[None, [2], [3], [4], [6]], - ListArray[[c], None, None, None, None], + Int64[30, 40, 50, 70], + Timestamp(Nanosecond, None)[None, None, None, None], + NullArray(4), + NullArray(4), + ListArray[[2], [3], [4], [6]], + ListArray[None, None, None, None], ]\ ", ); @@ -1983,11 +2021,11 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 10, 20, 30, 40, 50, 60, 65, 70], - Timestamp(Nanosecond, None)[None, 1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, 1970-01-01 00:00:00.000000060, 1970-01-01 00:00:00.000000065, 1970-01-01 00:00:00.000000070], - ListArray[[], None, None, [2], [3], [4], [], [], [6]], - ListArray[[], None, None, None, None, None, [], [], None], - ListArray[[], [{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [], [], [{x: 8, y: 8}]], + Int64[10, 20, 30, 40, 50, 60, 65, 70], + Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, 1970-01-01 00:00:00.000000060, 1970-01-01 00:00:00.000000065, 1970-01-01 00:00:00.000000070], + ListArray[None, None, [2], [3], [4], [], [], [6]], + ListArray[[c], [c], [c], [c], [c], [c], [c], [c]], + ListArray[[{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [], [], [{x: 8, y: 8}]], ]\ " ); @@ -2024,11 +2062,11 @@ mod tests { let expected = unindent::unindent( "\ [ - Int64[None, 10, 20, 30, 40, 50, 60, 65, 70], - Timestamp(Nanosecond, None)[None, 1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, 1970-01-01 00:00:00.000000060, 1970-01-01 00:00:00.000000065, 1970-01-01 00:00:00.000000070], - ListArray[[], None, None, [2], [3], [4], [], [], [6]], - ListArray[[], [c], [c], [c], [c], [c], [], [], [c]], - ListArray[[], [{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [], [], [{x: 8, y: 8}]], + Int64[10, 20, 30, 40, 50, 60, 65, 70], + Timestamp(Nanosecond, None)[1970-01-01 00:00:00.000000010, None, None, None, 1970-01-01 00:00:00.000000050, 1970-01-01 00:00:00.000000060, 1970-01-01 00:00:00.000000065, 1970-01-01 00:00:00.000000070], + ListArray[None, None, [2], [3], [4], [], [], [6]], + ListArray[[c], [c], [c], [c], [c], [c], [c], [c]], + ListArray[[{x: 0, y: 0}], [{x: 1, y: 1}], [{x: 2, y: 2}], [{x: 3, y: 3}], [{x: 4, y: 4}], [], [], [{x: 8, y: 8}]], ]\ " ); From fc1153221297e7973627886b85197a4046358950 Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 11 Oct 2024 16:39:07 +0200 Subject: [PATCH 3/5] implement new static semantics --- crates/store/re_dataframe/src/query.rs | 148 ++++++++++++++++--------- 1 file changed, 94 insertions(+), 54 deletions(-) diff --git a/crates/store/re_dataframe/src/query.rs b/crates/store/re_dataframe/src/query.rs index 584982f64550..54a6136ebf82 100644 --- a/crates/store/re_dataframe/src/query.rs +++ b/crates/store/re_dataframe/src/query.rs @@ -33,17 +33,6 @@ use crate::{QueryEngine, RecordBatch}; // --- -// TODO: -// -// * Default mode: -// * filtered_index = Some(something) -// * TimeInt::STATIC is never ever returned by default -// * Static values are repeated every row and override everything -// -// * Static mode: -// * filtered_index = None -// * TimeInt::STATIC is the only index value ever returned - // TODO(cmc): (no specific order) (should we make issues for these?) // * [x] basic thing working // * [x] custom selection @@ -98,6 +87,13 @@ struct QueryHandleState { /// See also [`QueryHandleState::arrow_schema`]. selected_contents: Vec<(usize, ColumnDescriptor)>, + /// This keeps track of the static data associated with each entry in `selected_contents`, if any. + /// + /// This is queried only once during init, and will override all cells that follow. + /// + /// `selected_contents`: [`QueryHandleState::selected_contents`] + selected_static_values: Vec>, + /// The actual index filter in use, since the user-specified one is optional. /// /// This just defaults to `Index::default()` if ther user hasn't specified any: the actual @@ -197,19 +193,20 @@ impl QueryHandle<'_> { // 4. Perform the query and keep track of all the relevant chunks. let query = { - let index_range = - if let Some(using_index_values) = self.query.using_index_values.as_ref() { - using_index_values - .first() - .and_then(|start| using_index_values.last().map(|end| (start, end))) - .map_or(ResolvedTimeRange::EMPTY, |(start, end)| { - ResolvedTimeRange::new(*start, *end) - }) - } else { - self.query - .filtered_index_range - .unwrap_or(ResolvedTimeRange::EVERYTHING) - }; + let index_range = if self.query.filtered_index.is_none() { + ResolvedTimeRange::EMPTY // static-only + } else if let Some(using_index_values) = self.query.using_index_values.as_ref() { + using_index_values + .first() + .and_then(|start| using_index_values.last().map(|end| (start, end))) + .map_or(ResolvedTimeRange::EMPTY, |(start, end)| { + ResolvedTimeRange::new(*start, *end) + }) + } else { + self.query + .filtered_index_range + .unwrap_or(ResolvedTimeRange::EVERYTHING) + }; RangeQuery::new(filtered_index, index_range) .keep_extra_timelines(true) // we want all the timelines we can get! @@ -279,45 +276,74 @@ impl QueryHandle<'_> { // 6. Collect all unique index values. // // Used to achieve ~O(log(n)) pagination. - let unique_index_values = - if let Some(using_index_values) = self.query.using_index_values.as_ref() { - using_index_values.iter().copied().collect_vec() - } else { - re_tracing::profile_scope!("index_values"); + let unique_index_values = if self.query.filtered_index.is_none() { + vec![TimeInt::STATIC] + } else if let Some(using_index_values) = self.query.using_index_values.as_ref() { + using_index_values + .iter() + .filter(|index_value| !index_value.is_static()) + .copied() + .collect_vec() + } else { + re_tracing::profile_scope!("index_values"); - let mut view_chunks = view_chunks.iter(); - let view_chunks = if let Some(view_pov_chunks_idx) = view_pov_chunks_idx { - Either::Left(view_chunks.nth(view_pov_chunks_idx).into_iter()) - } else { - Either::Right(view_chunks) - }; + let mut view_chunks = view_chunks.iter(); + let view_chunks = if let Some(view_pov_chunks_idx) = view_pov_chunks_idx { + Either::Left(view_chunks.nth(view_pov_chunks_idx).into_iter()) + } else { + Either::Right(view_chunks) + }; - let mut all_unique_index_values: BTreeSet = view_chunks - .flat_map(|chunks| { - chunks.iter().filter_map(|(_cursor, chunk)| { - if chunk.is_static() { - Some(Either::Left(std::iter::once(TimeInt::STATIC))) - } else { - chunk - .timelines() - .get(&filtered_index) - .map(|time_column| Either::Right(time_column.times())) - } - }) + let mut all_unique_index_values: BTreeSet = view_chunks + .flat_map(|chunks| { + chunks.iter().filter_map(|(_cursor, chunk)| { + chunk + .timelines() + .get(&filtered_index) + .map(|time_column| time_column.times()) }) - .flatten() - .collect(); + }) + .flatten() + .collect(); - if let Some(filtered_index_values) = self.query.filtered_index_values.as_ref() { - all_unique_index_values.retain(|time| filtered_index_values.contains(time)); - } + if let Some(filtered_index_values) = self.query.filtered_index_values.as_ref() { + all_unique_index_values.retain(|time| filtered_index_values.contains(time)); + } - all_unique_index_values.into_iter().collect_vec() - }; + all_unique_index_values + .into_iter() + .filter(|index_value| !index_value.is_static()) + .collect_vec() + }; + + let selected_static_values = { + re_tracing::profile_scope!("static_values"); + + selected_contents + .iter() + .map(|(_view_idx, descr)| match descr { + ColumnDescriptor::Time(_) => None, + ColumnDescriptor::Component(descr) => { + let query = + re_chunk::LatestAtQuery::new(Timeline::default(), TimeInt::STATIC); + + let results = self.engine.cache.latest_at( + self.engine.store, + &query, + &descr.entity_path, + [descr.component_name], + ); + + results.components.get(&descr.component_name).cloned() + } + }) + .collect_vec() + }; QueryHandleState { view_contents, selected_contents, + selected_static_values, filtered_index, arrow_schema, view_chunks, @@ -869,8 +895,22 @@ impl QueryHandle<'_> { .map(|streaming_state| streaming_state.map(StreamingJoinState::StreamingJoinState)) .collect_vec(); + // Static always wins, no matter what. + for (view_idx, streaming_state) in view_streaming_state.iter_mut().enumerate() { + if let static_state @ Some(_) = state + .selected_static_values + .get(view_idx) + .cloned() + .flatten() + .map(StreamingJoinState::Retrofilled) + { + *streaming_state = static_state; + } + } + match self.query.sparse_fill_strategy { SparseFillStrategy::None => {} + SparseFillStrategy::LatestAtGlobal => { // Everything that yielded `null` for the current iteration. let null_streaming_states = view_streaming_state From 7c6fbd07983490270de9b76b107d64216997584c Mon Sep 17 00:00:00 2001 From: Clement Rey Date: Fri, 11 Oct 2024 16:48:27 +0200 Subject: [PATCH 4/5] typo --- crates/store/re_dataframe/src/query.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/store/re_dataframe/src/query.rs b/crates/store/re_dataframe/src/query.rs index 54a6136ebf82..d90b27510e7a 100644 --- a/crates/store/re_dataframe/src/query.rs +++ b/crates/store/re_dataframe/src/query.rs @@ -96,7 +96,7 @@ struct QueryHandleState { /// The actual index filter in use, since the user-specified one is optional. /// - /// This just defaults to `Index::default()` if ther user hasn't specified any: the actual + /// This just defaults to `Index::default()` if the user hasn't specified any: the actual /// value is irrelevant since this means we are only concerned with static data anyway. filtered_index: Index, From 939c30e1f836764ab6bac96d0812ab3e26cafa0c Mon Sep 17 00:00:00 2001 From: Jeremy Leibs Date: Fri, 11 Oct 2024 21:43:49 +0200 Subject: [PATCH 5/5] Linter --- crates/store/re_dataframe/src/query.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/store/re_dataframe/src/query.rs b/crates/store/re_dataframe/src/query.rs index 43aa5622530e..529a16078694 100644 --- a/crates/store/re_dataframe/src/query.rs +++ b/crates/store/re_dataframe/src/query.rs @@ -1209,9 +1209,7 @@ mod tests { // static { - let query = QueryExpression { - ..Default::default() - }; + let query = QueryExpression::default(); eprintln!("{query:#?}:"); let query_handle = query_engine.query(query.clone());