From fd01be528680e6767acabe75e8b237f48ee57652 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Tue, 30 Jan 2024 16:08:25 +0800 Subject: [PATCH 01/13] rename `Executor:info` Signed-off-by: Richard Chien --- src/stream/src/executor/hash_agg.rs | 4 ++-- src/stream/src/executor/mod.rs | 2 +- src/stream/src/executor/mview/materialize.rs | 4 ++-- src/stream/src/executor/over_window/eowc.rs | 2 +- src/stream/src/executor/over_window/general.rs | 2 +- src/stream/src/executor/rearranged_chain.rs | 2 +- src/stream/src/executor/receiver.rs | 2 +- src/stream/src/executor/simple_agg.rs | 2 +- src/stream/src/executor/subscription.rs | 2 +- src/stream/src/executor/subtask.rs | 4 ++-- src/stream/src/executor/top_n/utils.rs | 2 +- src/stream/src/executor/union.rs | 2 +- src/stream/src/executor/watermark_filter.rs | 2 +- src/stream/src/executor/wrapper.rs | 4 ++-- src/stream/src/executor/wrapper/epoch_check.rs | 8 ++++---- src/stream/src/executor/wrapper/schema_check.rs | 4 ++-- src/stream/src/executor/wrapper/update_check.rs | 8 ++++---- 17 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index 5078fd0bc5154..60b29c1ce7931 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -213,7 +213,7 @@ impl Executor for HashAggExecutor { impl HashAggExecutor { pub fn new(args: AggExecutorArgs) -> StreamResult { - let input_info = args.input.info(); + let input_info = args.input.info_old(); let group_key_len = args.extra.group_key_indices.len(); // NOTE: we assume the prefix of table pk is exactly the group key @@ -620,7 +620,7 @@ impl HashAggExecutor { // TODO(rc): use something like a `ColumnMapping` type let group_key_invert_idx = { - let mut group_key_invert_idx = vec![None; input.info().schema.len()]; + let mut group_key_invert_idx = vec![None; input.info_old().schema.len()]; for (group_key_seq, group_key_idx) in this.group_key_indices.iter().enumerate() { group_key_invert_idx[*group_key_idx] = Some(group_key_seq); } diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index cb1d2a497ef8a..84b37671c9670 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -195,7 +195,7 @@ pub trait Executor: Send + 'static { } #[inline(always)] - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { let schema = self.schema().to_owned(); let pk_indices = self.pk_indices().to_owned(); let identity = self.identity().to_owned(); diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index 69e238a476f74..b84f6df60075e 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -446,7 +446,7 @@ impl Executor for MaterializeExecutor { &self.info.identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.info.clone() } } @@ -454,7 +454,7 @@ impl Executor for MaterializeExecutor { impl std::fmt::Debug for MaterializeExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("MaterializeExecutor") - .field("info", &self.info()) + .field("info", &self.info_old()) .field("arrange_key_indices", &self.arrange_key_indices) .finish() } diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 601a2f536aa00..73ba82e117a44 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -150,7 +150,7 @@ pub struct EowcOverWindowExecutorArgs { impl EowcOverWindowExecutor { pub fn new(args: EowcOverWindowExecutorArgs) -> Self { - let input_info = args.input.info(); + let input_info = args.input.info_old(); Self { input: args.input, diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index 55b0158ee89de..a3a239f18b3a1 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -163,7 +163,7 @@ pub struct OverWindowExecutorArgs { impl OverWindowExecutor { pub fn new(args: OverWindowExecutorArgs) -> Self { - let input_info = args.input.info(); + let input_info = args.input.info_old(); let input_schema = &input_info.schema; let has_unbounded_frame = args diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs index 1bb83009a203e..b0400698858bd 100644 --- a/src/stream/src/executor/rearranged_chain.rs +++ b/src/stream/src/executor/rearranged_chain.rs @@ -305,7 +305,7 @@ impl Executor for RearrangedChainExecutor { &self.info.identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.info.clone() } } diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index 273a3fcf1b339..437e51001b933 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -220,7 +220,7 @@ impl Executor for ReceiverExecutor { &self.info.identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.info.clone() } } diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index f957241a402c9..f8b49703c0f85 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -131,7 +131,7 @@ impl Executor for SimpleAggExecutor { impl SimpleAggExecutor { pub fn new(args: AggExecutorArgs) -> StreamResult { - let input_info = args.input.info(); + let input_info = args.input.info_old(); Ok(Self { input: args.input, inner: ExecutorInner { diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs index f1a2923c9085c..df167bc40c984 100644 --- a/src/stream/src/executor/subscription.rs +++ b/src/stream/src/executor/subscription.rs @@ -132,7 +132,7 @@ impl Executor for SubscriptionExecutor { &self.info.identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.info.clone() } } diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index 0130a73fc411a..4134b86d6d158 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -51,7 +51,7 @@ impl Executor for SubtaskRxExecutor { &self.info.identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.info.clone() } } @@ -67,7 +67,7 @@ pub fn wrap(input: BoxedExecutor, actor_id: ActorId) -> (SubtaskHandle, SubtaskR let rx_executor = SubtaskRxExecutor { info: ExecutorInfo { identity: "SubtaskRxExecutor".to_owned(), - ..input.info() + ..input.info_old() }, rx, }; diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index b2310f0d352b1..7b282ce8ccaec 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -97,7 +97,7 @@ where &self.inner.info().identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.inner.info().clone() } } diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index a3b3df6873e6b..7d882618811a5 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -65,7 +65,7 @@ impl Executor for UnionExecutor { &self.info.identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.info.clone() } } diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 43a6ba3add1ef..4ddd8345bdb91 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -98,7 +98,7 @@ impl Executor for WatermarkFilterExecutor { &self.info.identity } - fn info(&self) -> ExecutorInfo { + fn info_old(&self) -> ExecutorInfo { self.info.clone() } } diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index ebd3f6d9bfa73..e10d55ece341a 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -90,7 +90,7 @@ impl WrapperExecutor { impl Executor for WrapperExecutor { fn execute(self: Box) -> BoxedMessageStream { - let info = Arc::new(self.input.info()); + let info = Arc::new(self.input.info_old()); Self::wrap( self.enable_executor_row_count, info, @@ -101,7 +101,7 @@ impl Executor for WrapperExecutor { } fn execute_with_epoch(self: Box, epoch: u64) -> BoxedMessageStream { - let info = Arc::new(self.input.info()); + let info = Arc::new(self.input.info_old()); Self::wrap( self.enable_executor_row_count, info, diff --git a/src/stream/src/executor/wrapper/epoch_check.rs b/src/stream/src/executor/wrapper/epoch_check.rs index 732c67630a345..3f8e3a16d58f8 100644 --- a/src/stream/src/executor/wrapper/epoch_check.rs +++ b/src/stream/src/executor/wrapper/epoch_check.rs @@ -91,7 +91,7 @@ mod tests { tx.push_barrier(3, false); tx.push_barrier(4, false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let checked = epoch_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == 1); @@ -111,7 +111,7 @@ mod tests { tx.push_barrier(514, false); tx.push_barrier(114, false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let checked = epoch_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == 100); @@ -129,7 +129,7 @@ mod tests { tx.push_chunk(StreamChunk::default()); tx.push_barrier(114, false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let checked = epoch_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -139,7 +139,7 @@ mod tests { async fn test_empty() { let (_, mut source) = MockSource::channel(Default::default(), vec![]); source = source.stop_on_finish(false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let checked = epoch_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); assert!(checked.next().await.transpose().unwrap().is_none()); diff --git a/src/stream/src/executor/wrapper/schema_check.rs b/src/stream/src/executor/wrapper/schema_check.rs index 462ef5962042e..c24ca03e77ee0 100644 --- a/src/stream/src/executor/wrapper/schema_check.rs +++ b/src/stream/src/executor/wrapper/schema_check.rs @@ -82,7 +82,7 @@ mod tests { )); tx.push_barrier(1, false); - let checked = schema_check(source.info().into(), source.boxed().execute()); + let checked = schema_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Chunk(_)); @@ -108,7 +108,7 @@ mod tests { )); tx.push_barrier(1, false); - let checked = schema_check(source.info().into(), source.boxed().execute()); + let checked = schema_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); } diff --git a/src/stream/src/executor/wrapper/update_check.rs b/src/stream/src/executor/wrapper/update_check.rs index 4049960845282..fe472ca61f103 100644 --- a/src/stream/src/executor/wrapper/update_check.rs +++ b/src/stream/src/executor/wrapper/update_check.rs @@ -76,7 +76,7 @@ mod tests { U+ 810", )); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -91,7 +91,7 @@ mod tests { U+ 114", )); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -108,7 +108,7 @@ mod tests { U- 1919810", )); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -119,7 +119,7 @@ mod tests { let (mut tx, source) = MockSource::channel(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info_old().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); From 0514cd1d716d6f8f8285b0bf04fe75d576ec0d34 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Tue, 30 Jan 2024 16:25:03 +0800 Subject: [PATCH 02/13] add new `Executor::info` method Signed-off-by: Richard Chien --- src/compute/tests/cdc_tests.rs | 24 +++++----- .../executor/backfill/arrangement_backfill.rs | 4 ++ .../src/executor/backfill/cdc/cdc_backfill.rs | 4 ++ .../executor/backfill/no_shuffle_backfill.rs | 4 ++ src/stream/src/executor/barrier_recv.rs | 4 ++ src/stream/src/executor/batch_query.rs | 4 ++ src/stream/src/executor/chain.rs | 4 ++ .../src/executor/dedup/append_only_dedup.rs | 4 ++ src/stream/src/executor/dml.rs | 4 ++ src/stream/src/executor/dynamic_filter.rs | 4 ++ src/stream/src/executor/expand.rs | 4 ++ src/stream/src/executor/filter.rs | 4 ++ src/stream/src/executor/flow_control.rs | 4 ++ src/stream/src/executor/hash_agg.rs | 4 ++ src/stream/src/executor/hash_join.rs | 4 ++ src/stream/src/executor/hop_window.rs | 4 ++ src/stream/src/executor/lookup.rs | 4 ++ src/stream/src/executor/lookup_union.rs | 4 ++ src/stream/src/executor/merge.rs | 4 ++ src/stream/src/executor/mod.rs | 2 + src/stream/src/executor/mview/materialize.rs | 4 ++ src/stream/src/executor/no_op.rs | 4 ++ src/stream/src/executor/now.rs | 4 ++ src/stream/src/executor/over_window/eowc.rs | 4 ++ .../src/executor/over_window/general.rs | 4 ++ src/stream/src/executor/project.rs | 4 ++ src/stream/src/executor/project_set.rs | 4 ++ src/stream/src/executor/rearranged_chain.rs | 4 ++ src/stream/src/executor/receiver.rs | 4 ++ src/stream/src/executor/row_id_gen.rs | 4 ++ src/stream/src/executor/simple_agg.rs | 4 ++ src/stream/src/executor/sink.rs | 4 ++ src/stream/src/executor/sort.rs | 4 ++ .../src/executor/source/fetch_executor.rs | 4 ++ .../src/executor/source/fs_source_executor.rs | 4 ++ .../src/executor/source/list_executor.rs | 4 ++ .../src/executor/source/source_executor.rs | 4 ++ .../src/executor/stateless_simple_agg.rs | 4 ++ src/stream/src/executor/subscription.rs | 4 ++ src/stream/src/executor/subtask.rs | 4 ++ src/stream/src/executor/temporal_join.rs | 4 ++ src/stream/src/executor/test_utils.rs | 45 ++++++++++++------- src/stream/src/executor/top_n/utils.rs | 4 ++ src/stream/src/executor/union.rs | 4 ++ src/stream/src/executor/utils.rs | 4 ++ src/stream/src/executor/values.rs | 4 ++ src/stream/src/executor/watermark_filter.rs | 4 ++ src/stream/src/executor/wrapper.rs | 4 ++ 48 files changed, 225 insertions(+), 26 deletions(-) diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs index 2f1feec23cd46..897019bc75925 100644 --- a/src/compute/tests/cdc_tests.rs +++ b/src/compute/tests/cdc_tests.rs @@ -53,11 +53,7 @@ use risingwave_stream::executor::{ pub struct MockOffsetGenExecutor { upstream: Option, - schema: Schema, - - pk_indices: PkIndices, - - identity: String, + info: ExecutorInfo, start_offset: u32, } @@ -66,9 +62,11 @@ impl MockOffsetGenExecutor { pub fn new(upstream: StreamBoxedExecutor, schema: Schema, pk_indices: PkIndices) -> Self { Self { upstream: Some(upstream), - schema, - pk_indices, - identity: "MockOffsetGenExecutor".to_string(), + info: ExecutorInfo { + schema, + pk_indices, + identity: "MockOffsetGenExecutor".to_string(), + }, start_offset: 0, } } @@ -137,15 +135,19 @@ impl Executor for MockOffsetGenExecutor { } fn schema(&self) -> &Schema { - &self.schema + &self.info.schema } fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.pk_indices + &self.info.pk_indices } fn identity(&self) -> &str { - &self.identity + &self.info.identity + } + + fn info(&self) -> &ExecutorInfo { + &self.info } } diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index a7d2ab5c863eb..8f2f0f13b40c9 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -651,4 +651,8 @@ where fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index 75a421c1319fb..8ff44ece58825 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -605,6 +605,10 @@ impl Executor for CdcBackfillExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index 6d95a3ceba522..dce1fa50d123a 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -758,4 +758,8 @@ where fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } diff --git a/src/stream/src/executor/barrier_recv.rs b/src/stream/src/executor/barrier_recv.rs index 66480ef1cb591..d8c740539216e 100644 --- a/src/stream/src/executor/barrier_recv.rs +++ b/src/stream/src/executor/barrier_recv.rs @@ -81,6 +81,10 @@ impl Executor for BarrierRecvExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs index 533a327432f43..e60fed4620952 100644 --- a/src/stream/src/executor/batch_query.rs +++ b/src/stream/src/executor/batch_query.rs @@ -93,6 +93,10 @@ where &self.info.identity } + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn execute_with_epoch(self: Box, epoch: u64) -> BoxedMessageStream { self.execute_inner(epoch).boxed() } diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index 0228f826a4bac..1166df0ab87a5 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -120,6 +120,10 @@ impl Executor for ChainExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index b2d17e9da638f..fbdd42d45d555 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -211,6 +211,10 @@ impl Executor for AppendOnlyDedupExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index 8e0a186e9c81c..5e72f3fbde4f6 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -293,6 +293,10 @@ impl Executor for DmlExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index 14cf7192bd4db..f77d5c0cacc30 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -509,6 +509,10 @@ impl Executor fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index cb8c45732d17f..f9d0391df393d 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -86,6 +86,10 @@ impl Executor for ExpandExecutor { &self.info.identity } + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index e51307b2ffee0..62012608a4f29 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -148,6 +148,10 @@ impl Executor for FilterExecutor { &self.info.identity } + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs index 53d0f15a602ae..e9d4670e44fb0 100644 --- a/src/stream/src/executor/flow_control.rs +++ b/src/stream/src/executor/flow_control.rs @@ -141,6 +141,10 @@ impl Executor for FlowControlExecutor { self.input.pk_indices() } + fn info(&self) -> &ExecutorInfo { + &self.input.info() + } + fn identity(&self) -> &str { &self.identity } diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index 60b29c1ce7931..9391d18cffc51 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -209,6 +209,10 @@ impl Executor for HashAggExecutor { fn identity(&self) -> &str { &self.inner.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.inner.info + } } impl HashAggExecutor { diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index af35e7c7b9603..289fc37603a78 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -293,6 +293,10 @@ impl Executor for HashJoi fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } struct HashJoinChunkBuilder { diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs index 4bcca4d593072..a157c42bab21a 100644 --- a/src/stream/src/executor/hop_window.rs +++ b/src/stream/src/executor/hop_window.rs @@ -84,6 +84,10 @@ impl Executor for HopWindowExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl HopWindowExecutor { diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs index 87cc163ea3dff..48b10c0ad5e31 100644 --- a/src/stream/src/executor/lookup.rs +++ b/src/stream/src/executor/lookup.rs @@ -99,4 +99,8 @@ impl Executor for LookupExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index c52d562277459..eda86136ac6a3 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -70,6 +70,10 @@ impl Executor for LookupUnionExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl LookupUnionExecutor { diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index 145a937fcdb2e..b88c13b568173 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -261,6 +261,10 @@ impl Executor for MergeExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } /// A stream for merging messages from multiple upstreams. diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index 84b37671c9670..0921fee6d8d67 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -206,6 +206,8 @@ pub trait Executor: Send + 'static { } } + fn info(&self) -> &ExecutorInfo; + fn boxed(self) -> BoxedExecutor where Self: Sized + Send + 'static, diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index b84f6df60075e..49712cefdfe47 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -449,6 +449,10 @@ impl Executor for MaterializeExecutor { fn info_old(&self) -> ExecutorInfo { self.info.clone() } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl std::fmt::Debug for MaterializeExecutor { diff --git a/src/stream/src/executor/no_op.rs b/src/stream/src/executor/no_op.rs index f116eced8864d..b002fbfde6595 100644 --- a/src/stream/src/executor/no_op.rs +++ b/src/stream/src/executor/no_op.rs @@ -52,4 +52,8 @@ impl Executor for NoOpExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index 853411089eaba..8bcf1db20cef9 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -174,6 +174,10 @@ impl Executor for NowExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 73ba82e117a44..00bbab2c9dd0d 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -133,6 +133,10 @@ impl Executor for EowcOverWindowExecutor { fn identity(&self) -> &str { &self.inner.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.inner.info + } } pub struct EowcOverWindowExecutorArgs { diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index a3a239f18b3a1..4afdf370f08c3 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -113,6 +113,10 @@ impl Executor for OverWindowExecutor { fn identity(&self) -> &str { &self.inner.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.inner.info + } } impl ExecutorInner { diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index efc381560e75d..ee9765ea4bbea 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -99,6 +99,10 @@ impl Executor for ProjectExecutor { &self.inner.info.identity } + fn info(&self) -> &ExecutorInfo { + &self.inner.info + } + fn execute(self: Box) -> BoxedMessageStream { self.inner.execute(self.input).boxed() } diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs index 5167a71db2cb8..f2b778846fa4d 100644 --- a/src/stream/src/executor/project_set.rs +++ b/src/stream/src/executor/project_set.rs @@ -106,6 +106,10 @@ impl Executor for ProjectSetExecutor { fn identity(&self) -> &str { &self.inner.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.inner.info + } } impl Inner { diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs index b0400698858bd..1377011dd074d 100644 --- a/src/stream/src/executor/rearranged_chain.rs +++ b/src/stream/src/executor/rearranged_chain.rs @@ -305,6 +305,10 @@ impl Executor for RearrangedChainExecutor { &self.info.identity } + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn info_old(&self) -> ExecutorInfo { self.info.clone() } diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index 437e51001b933..596dd5cdaa036 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -223,6 +223,10 @@ impl Executor for ReceiverExecutor { fn info_old(&self) -> ExecutorInfo { self.info.clone() } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index fe0ed6d908925..346b40ca8f257 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -142,6 +142,10 @@ impl Executor for RowIdGenExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index f8b49703c0f85..fb207b4426c1b 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -127,6 +127,10 @@ impl Executor for SimpleAggExecutor { fn identity(&self) -> &str { &self.inner.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.inner.info + } } impl SimpleAggExecutor { diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index 1f8de0bb4dd92..9fc8dfd635d29 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -418,6 +418,10 @@ impl Executor for SinkExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index 786eabdeabbf5..3a634f65fdb12 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -72,6 +72,10 @@ impl Executor for SortExecutor { fn identity(&self) -> &str { &self.inner.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.inner.info + } } impl SortExecutor { diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs index c29afda3dcae5..bb1098dd6878f 100644 --- a/src/stream/src/executor/source/fetch_executor.rs +++ b/src/stream/src/executor/source/fetch_executor.rs @@ -360,6 +360,10 @@ impl Executor for FsFetchExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl Debug for FsFetchExecutor { diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index cb473b82d4c4a..376b59abdead4 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -492,6 +492,10 @@ impl Executor for FsSourceExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl Debug for FsSourceExecutor { diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs index f2a6e47ae8641..90082628c56ab 100644 --- a/src/stream/src/executor/source/list_executor.rs +++ b/src/stream/src/executor/source/list_executor.rs @@ -209,6 +209,10 @@ impl Executor for FsListExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl Debug for FsListExecutor { diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index 4eacd2a01709d..57110c6d91fbf 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -677,6 +677,10 @@ impl Executor for SourceExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl Debug for SourceExecutor { diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index 8a6334b7743b5..482b5e1cde5e4 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -51,6 +51,10 @@ impl Executor for StatelessSimpleAggExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl StatelessSimpleAggExecutor { diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs index df167bc40c984..a0c11afb6b8d8 100644 --- a/src/stream/src/executor/subscription.rs +++ b/src/stream/src/executor/subscription.rs @@ -135,4 +135,8 @@ impl Executor for SubscriptionExecutor { fn info_old(&self) -> ExecutorInfo { self.info.clone() } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index 4134b86d6d158..9472b891ae2a3 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -54,6 +54,10 @@ impl Executor for SubtaskRxExecutor { fn info_old(&self) -> ExecutorInfo { self.info.clone() } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } /// Wrap an executor into a subtask and a thin receiver executor, connected by a channel with a diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs index 099abe658f615..286867fae5a0f 100644 --- a/src/stream/src/executor/temporal_join.rs +++ b/src/stream/src/executor/temporal_join.rs @@ -506,4 +506,8 @@ impl Executor fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index cc7505164d154..922f4d9752ef9 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -21,8 +21,8 @@ use tokio::sync::mpsc; use super::error::StreamExecutorError; use super::{ - Barrier, BoxedMessageStream, Executor, Message, MessageStream, PkIndices, StreamChunk, - StreamExecutorResult, Watermark, + Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, MessageStream, PkIndices, + StreamChunk, StreamExecutorResult, Watermark, }; pub mod prelude { @@ -44,8 +44,7 @@ pub mod prelude { } pub struct MockSource { - schema: Schema, - pk_indices: PkIndices, + info: ExecutorInfo, rx: mpsc::UnboundedReceiver, /// Whether to send a `Stop` barrier on stream finish. @@ -108,22 +107,34 @@ impl MessageSender { impl std::fmt::Debug for MockSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("MockSource") - .field("schema", &self.schema) - .field("pk_indices", &self.pk_indices) + .field("schema", &self.info.schema) + .field("pk_indices", &self.info.pk_indices) .finish() } } impl MockSource { + fn new( + schema: Schema, + pk_indices: PkIndices, + rx: mpsc::UnboundedReceiver, + stop_on_finish: bool, + ) -> Self { + Self { + info: ExecutorInfo { + schema, + pk_indices, + identity: "MockSource".to_string(), + }, + rx, + stop_on_finish, + } + } + #[allow(dead_code)] pub fn channel(schema: Schema, pk_indices: PkIndices) -> (MessageSender, Self) { let (tx, rx) = mpsc::unbounded_channel(); - let source = Self { - schema, - pk_indices, - rx, - stop_on_finish: true, - }; + let source = Self::new(schema, pk_indices, rx, true); (MessageSender(tx), source) } @@ -174,15 +185,19 @@ impl Executor for MockSource { } fn schema(&self) -> &Schema { - &self.schema + &self.info.schema } fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.pk_indices + &self.info.pk_indices } fn identity(&self) -> &str { - "MockSource" + &self.info.identity + } + + fn info(&self) -> &ExecutorInfo { + &self.info } } diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index 7b282ce8ccaec..716802516988c 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -100,6 +100,10 @@ where fn info_old(&self) -> ExecutorInfo { self.inner.info().clone() } + + fn info(&self) -> &ExecutorInfo { + &self.inner.info() + } } impl TopNExecutorWrapper diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index 7d882618811a5..c12771ff19252 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -68,6 +68,10 @@ impl Executor for UnionExecutor { fn info_old(&self) -> ExecutorInfo { self.info.clone() } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[pin_project] diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs index 8ff9f8f29576d..a786cd4ecde68 100644 --- a/src/stream/src/executor/utils.rs +++ b/src/stream/src/executor/utils.rs @@ -47,6 +47,10 @@ impl Executor for DummyExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } pub(crate) struct ActorInputMetrics { diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index 2a927f1a3a780..288865b2759fd 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -151,6 +151,10 @@ impl Executor for ValuesExecutor { fn identity(&self) -> &str { &self.info.identity } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } #[cfg(test)] diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 4ddd8345bdb91..46d9c7cb7a72b 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -101,6 +101,10 @@ impl Executor for WatermarkFilterExecutor { fn info_old(&self) -> ExecutorInfo { self.info.clone() } + + fn info(&self) -> &ExecutorInfo { + &self.info + } } impl WatermarkFilterExecutor { diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index e10d55ece341a..1cd7172fa6568 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -122,4 +122,8 @@ impl Executor for WrapperExecutor { fn identity(&self) -> &str { self.input.identity() } + + fn info(&self) -> &ExecutorInfo { + &self.input.info() + } } From f8e8c8c24f5b045bd5b099f65187db13f04f2e54 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Tue, 30 Jan 2024 16:40:53 +0800 Subject: [PATCH 03/13] impl `pk_indices`, `identity`, `schema` via `info` Signed-off-by: Richard Chien --- .../executor/backfill/arrangement_backfill.rs | 20 +++-------- .../src/executor/backfill/cdc/cdc_backfill.rs | 20 +++-------- .../executor/backfill/no_shuffle_backfill.rs | 20 +++-------- src/stream/src/executor/barrier_recv.rs | 20 +++-------- src/stream/src/executor/batch_query.rs | 20 +++-------- src/stream/src/executor/chain.rs | 20 +++-------- .../src/executor/dedup/append_only_dedup.rs | 20 +++-------- src/stream/src/executor/dml.rs | 20 +++-------- src/stream/src/executor/dynamic_filter.rs | 20 +++-------- src/stream/src/executor/expand.rs | 12 ------- src/stream/src/executor/filter.rs | 12 ------- src/stream/src/executor/flow_control.rs | 16 ++------- src/stream/src/executor/hash_agg.rs | 20 +++-------- src/stream/src/executor/hash_join.rs | 20 +++-------- src/stream/src/executor/hop_window.rs | 20 +++-------- src/stream/src/executor/lookup.rs | 20 +++-------- src/stream/src/executor/lookup_union.rs | 20 +++-------- src/stream/src/executor/merge.rs | 20 +++-------- src/stream/src/executor/mod.rs | 33 ++++++++++--------- src/stream/src/executor/mview/materialize.rs | 24 +++----------- src/stream/src/executor/no_op.rs | 20 +++-------- src/stream/src/executor/now.rs | 20 +++-------- src/stream/src/executor/over_window/eowc.rs | 20 +++-------- .../src/executor/over_window/general.rs | 20 +++-------- src/stream/src/executor/project.rs | 12 ------- src/stream/src/executor/project_set.rs | 20 +++-------- src/stream/src/executor/rearranged_chain.rs | 20 ++--------- src/stream/src/executor/receiver.rs | 24 +++----------- src/stream/src/executor/row_id_gen.rs | 20 +++-------- src/stream/src/executor/simple_agg.rs | 20 +++-------- src/stream/src/executor/sink.rs | 20 +++-------- src/stream/src/executor/sort.rs | 20 +++-------- .../src/executor/source/fetch_executor.rs | 20 +++-------- .../src/executor/source/fs_source_executor.rs | 20 +++-------- .../src/executor/source/list_executor.rs | 20 +++-------- .../src/executor/source/source_executor.rs | 20 +++-------- .../src/executor/stateless_simple_agg.rs | 20 +++-------- src/stream/src/executor/subscription.rs | 24 +++----------- src/stream/src/executor/subtask.rs | 24 +++----------- src/stream/src/executor/temporal_join.rs | 20 +++-------- src/stream/src/executor/test_utils.rs | 20 +++-------- src/stream/src/executor/top_n/utils.rs | 24 +++----------- src/stream/src/executor/union.rs | 24 +++----------- src/stream/src/executor/utils.rs | 20 +++-------- src/stream/src/executor/values.rs | 20 +++-------- src/stream/src/executor/watermark_filter.rs | 24 +++----------- src/stream/src/executor/wrapper.rs | 20 +++-------- 47 files changed, 186 insertions(+), 767 deletions(-) diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 8f2f0f13b40c9..5f556452971d7 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -636,23 +636,11 @@ where S: StateStore, SD: ValueRowSerde, { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index 8ff44ece58825..1782a5451afd2 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -590,25 +590,13 @@ fn get_rw_columns(schema: &Schema) -> Vec { } impl Executor for CdcBackfillExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index dce1fa50d123a..d4eeb59d83c69 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -743,23 +743,11 @@ impl Executor for BackfillExecutor where S: StateStore, { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } diff --git a/src/stream/src/executor/barrier_recv.rs b/src/stream/src/executor/barrier_recv.rs index d8c740539216e..4d51f66322647 100644 --- a/src/stream/src/executor/barrier_recv.rs +++ b/src/stream/src/executor/barrier_recv.rs @@ -59,6 +59,10 @@ impl BarrierRecvExecutor { } impl Executor for BarrierRecvExecutor { + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn execute(self: Box) -> BoxedMessageStream { UnboundedReceiverStream::new(self.barrier_receiver) .map(|barrier| Ok(Message::Barrier(barrier))) @@ -69,22 +73,6 @@ impl Executor for BarrierRecvExecutor { })) .boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> &ExecutorInfo { - &self.info - } } #[cfg(test)] diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs index e60fed4620952..ea32b6de577d3 100644 --- a/src/stream/src/executor/batch_query.rs +++ b/src/stream/src/executor/batch_query.rs @@ -77,26 +77,14 @@ impl Executor for BatchQueryExecutor where S: StateStore, { - fn execute(self: Box) -> super::BoxedMessageStream { - unreachable!("should call `execute_with_epoch`") - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + fn execute(self: Box) -> super::BoxedMessageStream { + unreachable!("should call `execute_with_epoch`") + } + fn execute_with_epoch(self: Box, epoch: u64) -> BoxedMessageStream { self.execute_inner(epoch).boxed() } diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index 1166df0ab87a5..fd10163aae5fa 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -105,25 +105,13 @@ impl ChainExecutor { } impl Executor for ChainExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + self.execute_inner().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index fbdd42d45d555..303dabe147d6a 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -196,25 +196,13 @@ impl AppendOnlyDedupExecutor { } impl Executor for AppendOnlyDedupExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.executor_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.executor_inner().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index 5e72f3fbde4f6..a650479fae30b 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -278,25 +278,13 @@ impl DmlExecutor { } impl Executor for DmlExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index f77d5c0cacc30..fb97f41226672 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -494,25 +494,13 @@ impl DynamicFilterExecutor Executor for DynamicFilterExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.into_stream().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index f9d0391df393d..f72ff9e14f326 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -74,18 +74,6 @@ impl Debug for ExpandExecutor { } impl Executor for ExpandExecutor { - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index 62012608a4f29..e27c354811f58 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -136,18 +136,6 @@ impl Debug for FilterExecutor { } impl Executor for FilterExecutor { - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs index e9d4670e44fb0..250d5d89b70b3 100644 --- a/src/stream/src/executor/flow_control.rs +++ b/src/stream/src/executor/flow_control.rs @@ -129,23 +129,11 @@ impl Debug for FlowControlExecutor { } impl Executor for FlowControlExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - self.input.schema() - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - self.input.pk_indices() - } - fn info(&self) -> &ExecutorInfo { &self.input.info() } - fn identity(&self) -> &str { - &self.identity + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() } } diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index 9391d18cffc51..d4f8f67466a2a 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -194,25 +194,13 @@ impl ExecutionStats { } impl Executor for HashAggExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.inner.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } impl HashAggExecutor { diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index 289fc37603a78..f807371f5dd2b 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -278,25 +278,13 @@ impl std::fmt::Debug } impl Executor for HashJoinExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.into_stream().boxed() + } } struct HashJoinChunkBuilder { diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs index a157c42bab21a..43a05fe4ff04b 100644 --- a/src/stream/src/executor/hop_window.rs +++ b/src/stream/src/executor/hop_window.rs @@ -69,25 +69,13 @@ impl HopWindowExecutor { } impl Executor for HopWindowExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + self.execute_inner().boxed() + } } impl HopWindowExecutor { diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs index 48b10c0ad5e31..1a684c377b0ab 100644 --- a/src/stream/src/executor/lookup.rs +++ b/src/stream/src/executor/lookup.rs @@ -84,23 +84,11 @@ pub struct LookupExecutor { #[async_trait] impl Executor for LookupExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index eda86136ac6a3..bd3500fe3c6a8 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -55,25 +55,13 @@ impl LookupUnionExecutor { #[async_trait] impl Executor for LookupUnionExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } impl LookupUnionExecutor { diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index b88c13b568173..e0b5b69da2480 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -246,25 +246,13 @@ impl MergeExecutor { } impl Executor for MergeExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } /// A stream for merging messages from multiple upstreams. diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index 0921fee6d8d67..cbef3293f96b7 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -165,30 +165,23 @@ pub trait MessageStream = futures::Stream + Send; /// Static information of an executor. #[derive(Debug, Default, Clone)] pub struct ExecutorInfo { - /// See [`Executor::schema`]. + /// The schema of the OUTPUT of the executor. pub schema: Schema, - /// See [`Executor::pk_indices`]. + /// The primary key indices of the OUTPUT of the executor. + /// Schema is used by both OLAP and streaming, therefore + /// pk indices are maintained independently. pub pk_indices: PkIndices, - /// See [`Executor::identity`]. + /// Identity of the executor. pub identity: String, } /// `Executor` supports handling of control messages. pub trait Executor: Send + 'static { - fn execute(self: Box) -> BoxedMessageStream; - - /// Return the schema of the OUTPUT of the executor. - fn schema(&self) -> &Schema; - - /// Return the primary key indices of the OUTPUT of the executor. - /// Schema is used by both OLAP and streaming, therefore - /// pk indices are maintained independently. - fn pk_indices(&self) -> PkIndicesRef<'_>; + fn info(&self) -> &ExecutorInfo; - /// Identity of the executor. - fn identity(&self) -> &str; + fn execute(self: Box) -> BoxedMessageStream; fn execute_with_epoch(self: Box, _epoch: u64) -> BoxedMessageStream { self.execute() @@ -206,7 +199,17 @@ pub trait Executor: Send + 'static { } } - fn info(&self) -> &ExecutorInfo; + fn schema(&self) -> &Schema { + &self.info().schema + } + + fn pk_indices(&self) -> PkIndicesRef<'_> { + &self.info().pk_indices + } + + fn identity(&self) -> &str { + &self.info().identity + } fn boxed(self) -> BoxedExecutor where diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index 49712cefdfe47..3f4ddf8be98d4 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -430,29 +430,13 @@ impl MaterializeBuffer { } } impl Executor for MaterializeExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info_old(&self) -> ExecutorInfo { - self.info.clone() - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } impl std::fmt::Debug for MaterializeExecutor { diff --git a/src/stream/src/executor/no_op.rs b/src/stream/src/executor/no_op.rs index b002fbfde6595..e97842414b995 100644 --- a/src/stream/src/executor/no_op.rs +++ b/src/stream/src/executor/no_op.rs @@ -37,23 +37,11 @@ impl NoOpExecutor { } impl Executor for NoOpExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.input.execute() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.input.execute() + } } diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index 8bcf1db20cef9..5d5bd594ce594 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -159,25 +159,13 @@ impl NowExecutor { } impl Executor for NowExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.into_stream().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 00bbab2c9dd0d..2b0ef95fa3edd 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -118,25 +118,13 @@ struct ExecutionVars { } impl Executor for EowcOverWindowExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.executor_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.inner.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.executor_inner().boxed() + } } pub struct EowcOverWindowExecutorArgs { diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index 4afdf370f08c3..a54b1437073d4 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -98,25 +98,13 @@ struct ExecutionStats { } impl Executor for OverWindowExecutor { - fn execute(self: Box) -> crate::executor::BoxedMessageStream { - self.executor_inner().boxed() - } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> crate::executor::PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.inner.info } + + fn execute(self: Box) -> crate::executor::BoxedMessageStream { + self.executor_inner().boxed() + } } impl ExecutorInner { diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index ee9765ea4bbea..e16a35872857c 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -87,18 +87,6 @@ impl Debug for ProjectExecutor { } impl Executor for ProjectExecutor { - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.inner.info } diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs index f2b778846fa4d..1fe07bc5908c0 100644 --- a/src/stream/src/executor/project_set.rs +++ b/src/stream/src/executor/project_set.rs @@ -91,25 +91,13 @@ impl Debug for ProjectSetExecutor { } impl Executor for ProjectSetExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - self.inner.execute(self.input).boxed() - } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.inner.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + self.inner.execute(self.input).boxed() + } } impl Inner { diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs index 1377011dd074d..13ab898732e5e 100644 --- a/src/stream/src/executor/rearranged_chain.rs +++ b/src/stream/src/executor/rearranged_chain.rs @@ -289,28 +289,12 @@ impl RearrangedChainExecutor { } impl Executor for RearrangedChainExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } - fn info_old(&self) -> ExecutorInfo { - self.info.clone() + fn execute(self: Box) -> super::BoxedMessageStream { + self.execute_inner().boxed() } } diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index 596dd5cdaa036..c88e67f462a75 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -111,6 +111,10 @@ impl ReceiverExecutor { } impl Executor for ReceiverExecutor { + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn execute(mut self: Box) -> BoxedMessageStream { let actor_id = self.actor_context.id; @@ -207,26 +211,6 @@ impl Executor for ReceiverExecutor { stream.boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info_old(&self) -> ExecutorInfo { - self.info.clone() - } - - fn info(&self) -> &ExecutorInfo { - &self.info - } } #[cfg(test)] diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index 346b40ca8f257..16eafd479a5a2 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -127,25 +127,13 @@ impl RowIdGenExecutor { } impl Executor for RowIdGenExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + self.execute_inner().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index fb207b4426c1b..639474d5ff375 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -112,25 +112,13 @@ struct ExecutionVars { } impl Executor for SimpleAggExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.inner.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } impl SimpleAggExecutor { diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index 9fc8dfd635d29..444b1cfcff979 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -403,25 +403,13 @@ impl SinkExecutor { } impl Executor for SinkExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner() + } } #[cfg(test)] diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index 3a634f65fdb12..f474c2d607d4f 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -57,25 +57,13 @@ struct ExecutionVars { } impl Executor for SortExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.executor_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.inner.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.executor_inner().boxed() + } } impl SortExecutor { diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs index bb1098dd6878f..ec80272863bd8 100644 --- a/src/stream/src/executor/source/fetch_executor.rs +++ b/src/stream/src/executor/source/fetch_executor.rs @@ -345,25 +345,13 @@ impl FsFetchExecutor { } impl Executor for FsFetchExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.into_stream().boxed() + } } impl Debug for FsFetchExecutor { diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index 376b59abdead4..f12e926fde55c 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -477,25 +477,13 @@ impl FsSourceExecutor { } impl Executor for FsSourceExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.into_stream().boxed() + } } impl Debug for FsSourceExecutor { diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs index 90082628c56ab..8f74703cc896d 100644 --- a/src/stream/src/executor/source/list_executor.rs +++ b/src/stream/src/executor/source/list_executor.rs @@ -194,25 +194,13 @@ impl FsListExecutor { } impl Executor for FsListExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.into_stream().boxed() + } } impl Debug for FsListExecutor { diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index 57110c6d91fbf..d6965a3b6884d 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -658,6 +658,10 @@ impl SourceExecutor { } impl Executor for SourceExecutor { + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn execute(self: Box) -> BoxedMessageStream { if self.stream_source_core.is_some() { self.execute_with_stream_source().boxed() @@ -665,22 +669,6 @@ impl Executor for SourceExecutor { self.execute_without_stream_source().boxed() } } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> &ExecutorInfo { - &self.info - } } impl Debug for SourceExecutor { diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index 482b5e1cde5e4..f4200888d41e3 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -36,25 +36,13 @@ pub struct StatelessSimpleAggExecutor { } impl Executor for StatelessSimpleAggExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } impl StatelessSimpleAggExecutor { diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs index a0c11afb6b8d8..a48f9c357b186 100644 --- a/src/stream/src/executor/subscription.rs +++ b/src/stream/src/executor/subscription.rs @@ -116,27 +116,11 @@ impl SubscriptionExecutor { } } impl Executor for SubscriptionExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info_old(&self) -> ExecutorInfo { - self.info.clone() - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } } diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index 9472b891ae2a3..89c6787d8e635 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -35,29 +35,13 @@ pub struct SubtaskRxExecutor { } impl Executor for SubtaskRxExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - ReceiverStream::new(self.rx).boxed() - } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info_old(&self) -> ExecutorInfo { - self.info.clone() - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + ReceiverStream::new(self.rx).boxed() + } } /// Wrap an executor into a subtask and a thin receiver executor, connected by a channel with a diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs index 286867fae5a0f..6576b9e6654ac 100644 --- a/src/stream/src/executor/temporal_join.rs +++ b/src/stream/src/executor/temporal_join.rs @@ -491,23 +491,11 @@ impl TemporalJoinExecutor impl Executor for TemporalJoinExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + self.into_stream().boxed() + } } diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index 922f4d9752ef9..74852fbf3c663 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -180,25 +180,13 @@ impl MockSource { } impl Executor for MockSource { - fn execute(self: Box) -> super::BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + self.execute_inner().boxed() + } } /// `row_nonnull` builds a `OwnedRow` with concrete values. diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index 716802516988c..0850cd004b2d0 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -81,29 +81,13 @@ impl Executor for TopNExecutorWrapper where E: TopNExecutorBase, { - fn execute(self: Box) -> BoxedMessageStream { - self.top_n_executor_execute().boxed() - } - - fn schema(&self) -> &Schema { - &self.inner.info().schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info().pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info().identity - } - - fn info_old(&self) -> ExecutorInfo { - self.inner.info().clone() - } - fn info(&self) -> &ExecutorInfo { &self.inner.info() } + + fn execute(self: Box) -> BoxedMessageStream { + self.top_n_executor_execute().boxed() + } } impl TopNExecutorWrapper diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index c12771ff19252..1b20b1e8f0a49 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -48,30 +48,14 @@ impl UnionExecutor { } impl Executor for UnionExecutor { + fn info(&self) -> &ExecutorInfo { + &self.info + } + fn execute(self: Box) -> BoxedMessageStream { let streams = self.inputs.into_iter().map(|e| e.execute()).collect(); merge(streams).boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info_old(&self) -> ExecutorInfo { - self.info.clone() - } - - fn info(&self) -> &ExecutorInfo { - &self.info - } } #[pin_project] diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs index a786cd4ecde68..859eee061b8c2 100644 --- a/src/stream/src/executor/utils.rs +++ b/src/stream/src/executor/utils.rs @@ -32,25 +32,13 @@ impl DummyExecutor { } impl Executor for DummyExecutor { - fn execute(self: Box) -> BoxedMessageStream { - futures::stream::pending().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + futures::stream::pending().boxed() + } } pub(crate) struct ActorInputMetrics { diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index 288865b2759fd..b918a36e1d89f 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -136,25 +136,13 @@ impl ValuesExecutor { } impl Executor for ValuesExecutor { - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> BoxedMessageStream { + self.into_stream().boxed() + } } #[cfg(test)] diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 46d9c7cb7a72b..37b2f1e41b564 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -82,29 +82,13 @@ impl WatermarkFilterExecutor { } impl Executor for WatermarkFilterExecutor { - fn execute(self: Box) -> super::BoxedMessageStream { - self.execute_inner().boxed() - } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info_old(&self) -> ExecutorInfo { - self.info.clone() - } - fn info(&self) -> &ExecutorInfo { &self.info } + + fn execute(self: Box) -> super::BoxedMessageStream { + self.execute_inner().boxed() + } } impl WatermarkFilterExecutor { diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index 1cd7172fa6568..c523a5e3c6c03 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -89,6 +89,10 @@ impl WrapperExecutor { } impl Executor for WrapperExecutor { + fn info(&self) -> &ExecutorInfo { + &self.input.info() + } + fn execute(self: Box) -> BoxedMessageStream { let info = Arc::new(self.input.info_old()); Self::wrap( @@ -110,20 +114,4 @@ impl Executor for WrapperExecutor { ) .boxed() } - - fn schema(&self) -> &Schema { - self.input.schema() - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - self.input.pk_indices() - } - - fn identity(&self) -> &str { - self.input.identity() - } - - fn info(&self) -> &ExecutorInfo { - &self.input.info() - } } From c1e6eb63c076e60c1f1fbfb04839fd56db0b5d28 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Tue, 30 Jan 2024 16:49:06 +0800 Subject: [PATCH 04/13] remove `Executor::info_old` Signed-off-by: Richard Chien --- src/stream/src/executor/hash_agg.rs | 4 ++-- src/stream/src/executor/mod.rs | 24 +++++-------------- src/stream/src/executor/mview/materialize.rs | 2 +- src/stream/src/executor/over_window/eowc.rs | 2 +- .../src/executor/over_window/general.rs | 2 +- src/stream/src/executor/simple_agg.rs | 2 +- src/stream/src/executor/subtask.rs | 2 +- src/stream/src/executor/wrapper.rs | 4 ++-- .../src/executor/wrapper/epoch_check.rs | 8 +++---- .../src/executor/wrapper/schema_check.rs | 4 ++-- .../src/executor/wrapper/update_check.rs | 8 +++---- 11 files changed, 25 insertions(+), 37 deletions(-) diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index d4f8f67466a2a..3a4f93b009195 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -205,7 +205,7 @@ impl Executor for HashAggExecutor { impl HashAggExecutor { pub fn new(args: AggExecutorArgs) -> StreamResult { - let input_info = args.input.info_old(); + let input_info = args.input.info().clone(); let group_key_len = args.extra.group_key_indices.len(); // NOTE: we assume the prefix of table pk is exactly the group key @@ -612,7 +612,7 @@ impl HashAggExecutor { // TODO(rc): use something like a `ColumnMapping` type let group_key_invert_idx = { - let mut group_key_invert_idx = vec![None; input.info_old().schema.len()]; + let mut group_key_invert_idx = vec![None; input.info().schema.len()]; for (group_key_seq, group_key_idx) in this.group_key_indices.iter().enumerate() { group_key_invert_idx[*group_key_idx] = Some(group_key_seq); } diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index cbef3293f96b7..6ea6272bcbfe4 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -181,24 +181,6 @@ pub struct ExecutorInfo { pub trait Executor: Send + 'static { fn info(&self) -> &ExecutorInfo; - fn execute(self: Box) -> BoxedMessageStream; - - fn execute_with_epoch(self: Box, _epoch: u64) -> BoxedMessageStream { - self.execute() - } - - #[inline(always)] - fn info_old(&self) -> ExecutorInfo { - let schema = self.schema().to_owned(); - let pk_indices = self.pk_indices().to_owned(); - let identity = self.identity().to_owned(); - ExecutorInfo { - schema, - pk_indices, - identity, - } - } - fn schema(&self) -> &Schema { &self.info().schema } @@ -211,6 +193,12 @@ pub trait Executor: Send + 'static { &self.info().identity } + fn execute(self: Box) -> BoxedMessageStream; + + fn execute_with_epoch(self: Box, _epoch: u64) -> BoxedMessageStream { + self.execute() + } + fn boxed(self) -> BoxedExecutor where Self: Sized + Send + 'static, diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index 3f4ddf8be98d4..faed426d2631f 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -442,7 +442,7 @@ impl Executor for MaterializeExecutor { impl std::fmt::Debug for MaterializeExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("MaterializeExecutor") - .field("info", &self.info_old()) + .field("info", &self.info) .field("arrange_key_indices", &self.arrange_key_indices) .finish() } diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 2b0ef95fa3edd..57a4263271658 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -142,7 +142,7 @@ pub struct EowcOverWindowExecutorArgs { impl EowcOverWindowExecutor { pub fn new(args: EowcOverWindowExecutorArgs) -> Self { - let input_info = args.input.info_old(); + let input_info = args.input.info().clone(); Self { input: args.input, diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index a54b1437073d4..3519e9d348471 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -155,7 +155,7 @@ pub struct OverWindowExecutorArgs { impl OverWindowExecutor { pub fn new(args: OverWindowExecutorArgs) -> Self { - let input_info = args.input.info_old(); + let input_info = args.input.info().clone(); let input_schema = &input_info.schema; let has_unbounded_frame = args diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index 639474d5ff375..65ef8f7be895c 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -123,7 +123,7 @@ impl Executor for SimpleAggExecutor { impl SimpleAggExecutor { pub fn new(args: AggExecutorArgs) -> StreamResult { - let input_info = args.input.info_old(); + let input_info = args.input.info().clone(); Ok(Self { input: args.input, inner: ExecutorInner { diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index 89c6787d8e635..e313b42d7d434 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -55,7 +55,7 @@ pub fn wrap(input: BoxedExecutor, actor_id: ActorId) -> (SubtaskHandle, SubtaskR let rx_executor = SubtaskRxExecutor { info: ExecutorInfo { identity: "SubtaskRxExecutor".to_owned(), - ..input.info_old() + ..input.info().clone() }, rx, }; diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index c523a5e3c6c03..adb82c4b3e4c7 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -94,7 +94,7 @@ impl Executor for WrapperExecutor { } fn execute(self: Box) -> BoxedMessageStream { - let info = Arc::new(self.input.info_old()); + let info = Arc::new(self.input.info().clone()); Self::wrap( self.enable_executor_row_count, info, @@ -105,7 +105,7 @@ impl Executor for WrapperExecutor { } fn execute_with_epoch(self: Box, epoch: u64) -> BoxedMessageStream { - let info = Arc::new(self.input.info_old()); + let info = Arc::new(self.input.info().clone()); Self::wrap( self.enable_executor_row_count, info, diff --git a/src/stream/src/executor/wrapper/epoch_check.rs b/src/stream/src/executor/wrapper/epoch_check.rs index 3f8e3a16d58f8..1477b3081f3f9 100644 --- a/src/stream/src/executor/wrapper/epoch_check.rs +++ b/src/stream/src/executor/wrapper/epoch_check.rs @@ -91,7 +91,7 @@ mod tests { tx.push_barrier(3, false); tx.push_barrier(4, false); - let checked = epoch_check(source.info_old().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == 1); @@ -111,7 +111,7 @@ mod tests { tx.push_barrier(514, false); tx.push_barrier(114, false); - let checked = epoch_check(source.info_old().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == 100); @@ -129,7 +129,7 @@ mod tests { tx.push_chunk(StreamChunk::default()); tx.push_barrier(114, false); - let checked = epoch_check(source.info_old().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -139,7 +139,7 @@ mod tests { async fn test_empty() { let (_, mut source) = MockSource::channel(Default::default(), vec![]); source = source.stop_on_finish(false); - let checked = epoch_check(source.info_old().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); assert!(checked.next().await.transpose().unwrap().is_none()); diff --git a/src/stream/src/executor/wrapper/schema_check.rs b/src/stream/src/executor/wrapper/schema_check.rs index c24ca03e77ee0..7d08c2dcc6b90 100644 --- a/src/stream/src/executor/wrapper/schema_check.rs +++ b/src/stream/src/executor/wrapper/schema_check.rs @@ -82,7 +82,7 @@ mod tests { )); tx.push_barrier(1, false); - let checked = schema_check(source.info_old().into(), source.boxed().execute()); + let checked = schema_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Chunk(_)); @@ -108,7 +108,7 @@ mod tests { )); tx.push_barrier(1, false); - let checked = schema_check(source.info_old().into(), source.boxed().execute()); + let checked = schema_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); } diff --git a/src/stream/src/executor/wrapper/update_check.rs b/src/stream/src/executor/wrapper/update_check.rs index fe472ca61f103..d880249de8dfb 100644 --- a/src/stream/src/executor/wrapper/update_check.rs +++ b/src/stream/src/executor/wrapper/update_check.rs @@ -76,7 +76,7 @@ mod tests { U+ 810", )); - let checked = update_check(source.info_old().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -91,7 +91,7 @@ mod tests { U+ 114", )); - let checked = update_check(source.info_old().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -108,7 +108,7 @@ mod tests { U- 1919810", )); - let checked = update_check(source.info_old().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -119,7 +119,7 @@ mod tests { let (mut tx, source) = MockSource::channel(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); - let checked = update_check(source.info_old().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.boxed().execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); From 3305d1fbb773a575787a9321d2ccb9c13167817d Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Tue, 30 Jan 2024 17:00:51 +0800 Subject: [PATCH 05/13] fix flow control executor identity Signed-off-by: Richard Chien --- src/stream/src/executor/flow_control.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs index 250d5d89b70b3..022796b251ebb 100644 --- a/src/stream/src/executor/flow_control.rs +++ b/src/stream/src/executor/flow_control.rs @@ -17,7 +17,6 @@ use std::num::NonZeroU32; use governor::clock::MonotonicClock; use governor::{Quota, RateLimiter}; -use risingwave_common::catalog::Schema; use super::*; @@ -31,9 +30,9 @@ use super::*; /// /// It is used to throttle problematic MVs that are consuming too much resources. pub struct FlowControlExecutor { + info: ExecutorInfo, input: BoxedExecutor, actor_ctx: ActorContextRef, - identity: String, rate_limit: Option, } @@ -49,9 +48,12 @@ impl FlowControlExecutor { input.identity().to_owned() }; Self { + info: ExecutorInfo { + identity, + ..input.info().clone() + }, input, actor_ctx, - identity, rate_limit, } } @@ -130,7 +132,7 @@ impl Debug for FlowControlExecutor { impl Executor for FlowControlExecutor { fn info(&self) -> &ExecutorInfo { - &self.input.info() + &self.info } fn execute(self: Box) -> BoxedMessageStream { From 006b977338826f282af3fad428915d0d15564299 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Tue, 30 Jan 2024 17:10:46 +0800 Subject: [PATCH 06/13] remove unused import Signed-off-by: Richard Chien --- src/stream/src/executor/backfill/arrangement_backfill.rs | 4 ++-- src/stream/src/executor/backfill/cdc/cdc_backfill.rs | 2 +- src/stream/src/executor/backfill/no_shuffle_backfill.rs | 4 ++-- src/stream/src/executor/barrier_recv.rs | 2 +- src/stream/src/executor/batch_query.rs | 2 +- src/stream/src/executor/chain.rs | 2 +- src/stream/src/executor/dedup/append_only_dedup.rs | 4 ++-- src/stream/src/executor/dml.rs | 6 +++--- src/stream/src/executor/dynamic_filter.rs | 3 +-- src/stream/src/executor/expand.rs | 2 +- src/stream/src/executor/filter.rs | 2 +- src/stream/src/executor/hash_agg.rs | 2 +- src/stream/src/executor/hash_join.rs | 5 ++--- src/stream/src/executor/lookup.rs | 4 ++-- src/stream/src/executor/lookup_union.rs | 2 +- src/stream/src/executor/merge.rs | 2 +- src/stream/src/executor/mview/materialize.rs | 4 ++-- src/stream/src/executor/no_op.rs | 4 ++-- src/stream/src/executor/now.rs | 4 ++-- src/stream/src/executor/over_window/eowc.rs | 4 ++-- src/stream/src/executor/project.rs | 2 +- src/stream/src/executor/project_set.rs | 4 ++-- src/stream/src/executor/rearranged_chain.rs | 2 +- src/stream/src/executor/receiver.rs | 8 ++++---- src/stream/src/executor/row_id_gen.rs | 4 ++-- src/stream/src/executor/sort.rs | 4 ++-- src/stream/src/executor/source/fetch_executor.rs | 2 +- src/stream/src/executor/source/fs_source_executor.rs | 2 +- src/stream/src/executor/source/list_executor.rs | 2 +- src/stream/src/executor/stateless_simple_agg.rs | 2 +- src/stream/src/executor/subscription.rs | 4 ++-- src/stream/src/executor/temporal_join.rs | 2 +- src/stream/src/executor/top_n/utils.rs | 4 ++-- src/stream/src/executor/union.rs | 2 +- src/stream/src/executor/utils.rs | 4 ++-- src/stream/src/executor/values.rs | 4 ++-- src/stream/src/executor/wrapper.rs | 4 ++-- 37 files changed, 59 insertions(+), 61 deletions(-) diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 5f556452971d7..40bf19ea32cf8 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -22,7 +22,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::bail; -use risingwave_common::catalog::Schema; + use risingwave_common::hash::{VirtualNode, VnodeBitmapExt}; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; use risingwave_common::util::iter_util::ZipEqDebug; @@ -41,7 +41,7 @@ use crate::executor::backfill::utils::{ use crate::executor::monitor::StreamingMetrics; use crate::executor::{ expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, - Message, PkIndicesRef, StreamExecutorError, + Message, StreamExecutorError, }; use crate::task::{ActorId, CreateMviewProgress}; diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index 1782a5451afd2..c2aad5d7e460b 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -46,7 +46,7 @@ use crate::executor::backfill::utils::{ use crate::executor::monitor::StreamingMetrics; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorError, StreamExecutorResult, + ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, }; use crate::task::CreateMviewProgress; diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index d4eeb59d83c69..86fb1766b8cca 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -20,7 +20,7 @@ use futures::stream::select_with_strategy; use futures::{stream, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{DataChunk, Op, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::Datum; @@ -40,7 +40,7 @@ use crate::executor::backfill::utils::{ use crate::executor::monitor::StreamingMetrics; use crate::executor::{ expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, - Message, Mutation, PkIndicesRef, StreamExecutorError, StreamExecutorResult, + Message, Mutation, StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; diff --git a/src/stream/src/executor/barrier_recv.rs b/src/stream/src/executor/barrier_recv.rs index 4d51f66322647..bb30850f69c69 100644 --- a/src/stream/src/executor/barrier_recv.rs +++ b/src/stream/src/executor/barrier_recv.rs @@ -19,7 +19,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ ActorContext, ActorContextRef, Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, - PkIndices, PkIndicesRef, StreamExecutorError, + PkIndices, StreamExecutorError, }; /// The executor only for receiving barrier from the meta service. It always resides in the leaves diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs index ea32b6de577d3..be476e20e670c 100644 --- a/src/stream/src/executor/batch_query.rs +++ b/src/stream/src/executor/batch_query.rs @@ -16,7 +16,7 @@ use await_tree::InstrumentAwait; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_hummock_sdk::HummockReadEpoch; use risingwave_storage::store::PrefetchOptions; use risingwave_storage::table::batch_table::storage_table::StorageTable; diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index fd10163aae5fa..5a08a9e56d185 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -14,7 +14,7 @@ use futures::StreamExt; use futures_async_stream::try_stream; -use risingwave_common::catalog::Schema; + use super::error::StreamExecutorError; use super::{expect_first_barrier, BoxedExecutor, Executor, ExecutorInfo, Message}; diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index 303dabe147d6a..78294d0f370ca 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -19,7 +19,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::BitmapBuilder; -use risingwave_common::catalog::Schema; + use risingwave_common::row::{OwnedRow, Row, RowExt}; use risingwave_storage::StateStore; @@ -30,7 +30,7 @@ use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorResult, + ExecutorInfo, Message, StreamExecutorResult, }; use crate::task::AtomicU64Ref; diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index a650479fae30b..35f14e5284703 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -19,7 +19,7 @@ use either::Either; use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; -use risingwave_common::catalog::{ColumnDesc, Schema, TableId, TableVersionId}; +use risingwave_common::catalog::{ColumnDesc, TableId, TableVersionId}; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::transaction::transaction_message::TxnMsg; use risingwave_dml::dml_manager::DmlManagerRef; @@ -27,7 +27,7 @@ use risingwave_dml::dml_manager::DmlManagerRef; use super::error::StreamExecutorError; use super::{ expect_first_barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, - Mutation, PkIndicesRef, + Mutation, }; use crate::common::StreamChunkBuilder; use crate::executor::stream_reader::StreamReaderWithPause; @@ -292,7 +292,7 @@ mod tests { use std::sync::Arc; use risingwave_common::array::StreamChunk; - use risingwave_common::catalog::{ColumnId, Field, INITIAL_TABLE_VERSION_ID}; + use risingwave_common::catalog::{ColumnId, Field, Schema, INITIAL_TABLE_VERSION_ID}; use risingwave_common::test_prelude::StreamChunkTestExt; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::types::DataType; diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index fb97f41226672..4c68125eb04aa 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -20,7 +20,7 @@ use futures_async_stream::try_stream; use risingwave_common::array::{Array, ArrayImpl, Op, StreamChunk}; use risingwave_common::bail; use risingwave_common::buffer::{Bitmap, BitmapBuilder}; -use risingwave_common::catalog::Schema; + use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{self, once, OwnedRow, OwnedRow as RowData, Row}; use risingwave_common::types::{DataType, Datum, DefaultOrd, ScalarImpl, ToDatumRef, ToOwnedDatum}; @@ -40,7 +40,6 @@ use super::error::StreamExecutorError; use super::monitor::StreamingMetrics; use super::{ ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, - PkIndicesRef, }; use crate::common::table::state_table::{StateTable, WatermarkCacheParameterizedStateTable}; use crate::common::StreamChunkBuilder; diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index f72ff9e14f326..dd36c559bffa5 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -17,7 +17,7 @@ use std::fmt::Debug; use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::{Array, I64Array}; -use risingwave_common::catalog::Schema; + use super::error::StreamExecutorError; use super::*; diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index e27c354811f58..33a92aa4216e3 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -17,7 +17,7 @@ use std::sync::Arc; use risingwave_common::array::{Array, ArrayImpl, Op, StreamChunk}; use risingwave_common::buffer::BitmapBuilder; -use risingwave_common::catalog::Schema; + use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index 3a4f93b009195..1bf50bb13e1b6 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -39,7 +39,7 @@ use super::aggregation::{ }; use super::sort_buffer::SortBuffer; use super::{ - expect_first_barrier, ActorContextRef, ExecutorInfo, PkIndicesRef, StreamExecutorResult, + expect_first_barrier, ActorContextRef, ExecutorInfo, StreamExecutorResult, Watermark, }; use crate::cache::{cache_may_stale, new_with_hasher, ManagedLruCache}; diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index f807371f5dd2b..998db81cc1b12 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -22,7 +22,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use multimap::MultiMap; use risingwave_common::array::{Op, RowRef, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_common::hash::{HashKey, NullBitmap}; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, DefaultOrd, ToOwnedDatum}; @@ -40,8 +40,7 @@ use super::managed_state::join::*; use super::monitor::StreamingMetrics; use super::watermark::*; use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, - PkIndicesRef, Watermark, + ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, Watermark, }; use crate::common::table::state_table::StateTable; use crate::common::JoinStreamChunkBuilder; diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs index 1a684c377b0ab..c8cb8848ba396 100644 --- a/src/stream/src/executor/lookup.rs +++ b/src/stream/src/executor/lookup.rs @@ -14,11 +14,11 @@ use async_trait::async_trait; use futures::StreamExt; -use risingwave_common::catalog::Schema; + use risingwave_common::types::DataType; use risingwave_storage::StateStore; -use crate::executor::{Barrier, BoxedMessageStream, Executor, PkIndicesRef}; +use crate::executor::{Barrier, BoxedMessageStream, Executor}; mod cache; mod sides; diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index bd3500fe3c6a8..0451da2b8ff3c 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -18,7 +18,7 @@ use futures::future::{join_all, select, Either}; use futures::{FutureExt, SinkExt, StreamExt}; use futures_async_stream::try_stream; use itertools::Itertools; -use risingwave_common::catalog::Schema; + use super::error::StreamExecutorError; use super::*; diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index e0b5b69da2480..1439fa64a0269 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -20,7 +20,7 @@ use anyhow::Context as _; use futures::stream::{FusedStream, FuturesUnordered, StreamFuture}; use futures::{pin_mut, Stream, StreamExt}; use futures_async_stream::try_stream; -use risingwave_common::catalog::Schema; + use tokio::time::Instant; use super::error::StreamExecutorError; diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index faed426d2631f..77ccf66b05dba 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -24,7 +24,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::Bitmap; -use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, Schema, TableId}; +use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, TableId}; use risingwave_common::row::{CompactedRow, RowDeserializer}; use risingwave_common::types::DataType; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; @@ -43,7 +43,7 @@ use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ expect_first_barrier, ActorContext, ActorContextRef, AddMutation, BoxedExecutor, - BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, PkIndicesRef, + BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, StreamExecutorResult, UpdateMutation, }; use crate::task::{ActorId, AtomicU64Ref}; diff --git a/src/stream/src/executor/no_op.rs b/src/stream/src/executor/no_op.rs index e97842414b995..4f4ac666aa732 100644 --- a/src/stream/src/executor/no_op.rs +++ b/src/stream/src/executor/no_op.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::catalog::Schema; + use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, PkIndicesRef, + ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, }; /// No-op executor directly forwards the input stream. Currently used to break the multiple edges in diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index 5d5bd594ce594..3bd8809158a32 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -18,7 +18,7 @@ use std::ops::Bound::Unbounded; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_common::row::{self, OwnedRow}; use risingwave_common::types::{DataType, Datum}; use risingwave_storage::StateStore; @@ -26,7 +26,7 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ - Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, PkIndicesRef, + Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, StreamExecutorError, Watermark, }; use crate::common::table::state_table::StateTable; diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 57a4263271658..42a1f41d6f4e3 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -20,7 +20,7 @@ use futures_async_stream::{for_await, try_stream}; use itertools::Itertools; use risingwave_common::array::stream_record::Record; use risingwave_common::array::{ArrayRef, Op, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_common::estimate_size::collections::EstimatedVecDeque; use risingwave_common::estimate_size::EstimateSize; use risingwave_common::row::{OwnedRow, Row, RowExt}; @@ -40,7 +40,7 @@ use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorError, StreamExecutorResult, + ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, }; use crate::task::AtomicU64Ref; diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index e16a35872857c..1a995d9c0ea2f 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -16,7 +16,7 @@ use std::fmt::{Debug, Formatter}; use multimap::MultiMap; use risingwave_common::array::StreamChunk; -use risingwave_common::catalog::Schema; + use risingwave_common::row::{Row, RowExt}; use risingwave_common::types::ToOwnedDatum; use risingwave_common::util::iter_util::ZipEqFast; diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs index 1fe07bc5908c0..35bb5c793e6fa 100644 --- a/src/stream/src/executor/project_set.rs +++ b/src/stream/src/executor/project_set.rs @@ -20,7 +20,7 @@ use futures_async_stream::try_stream; use multimap::MultiMap; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::bail; -use risingwave_common::catalog::Schema; + use risingwave_common::row::{Row, RowExt}; use risingwave_common::types::{DataType, Datum, DatumRef, ToOwnedDatum}; use risingwave_common::util::iter_util::ZipEqFast; @@ -29,7 +29,7 @@ use risingwave_expr::table_function::ProjectSetSelectItem; use super::error::StreamExecutorError; use super::{ - ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, PkIndicesRef, + ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, StreamExecutorResult, Watermark, }; use crate::common::StreamChunkBuilder; diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs index 13ab898732e5e..0c23bab02e1a1 100644 --- a/src/stream/src/executor/rearranged_chain.rs +++ b/src/stream/src/executor/rearranged_chain.rs @@ -19,7 +19,7 @@ use futures::stream::select_with_strategy; use futures::{stream, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; -use risingwave_common::catalog::Schema; + use super::error::StreamExecutorError; use super::{ diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index c88e67f462a75..fc84191d2ffc2 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -17,7 +17,6 @@ use anyhow::Context; use futures::StreamExt; use futures_async_stream::try_stream; use itertools::Itertools; -use risingwave_common::catalog::Schema; use tokio::time::Instant; use super::exchange::input::BoxedInput; @@ -25,9 +24,7 @@ use super::ActorContextRef; use crate::executor::exchange::input::new_input; use crate::executor::monitor::StreamingMetrics; use crate::executor::utils::ActorInputMetrics; -use crate::executor::{ - expect_first_barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, PkIndicesRef, -}; +use crate::executor::{expect_first_barrier, BoxedMessageStream, Executor, ExecutorInfo, Message}; use crate::task::{FragmentId, SharedContext}; /// `ReceiverExecutor` is used along with a channel. After creating a mpsc channel, /// there should be a `ReceiverExecutor` running in the background, so as to push @@ -89,6 +86,8 @@ impl ReceiverExecutor { #[cfg(test)] pub fn for_test(input: super::exchange::permit::Receiver) -> Self { + use risingwave_common::catalog::Schema; + use super::exchange::input::LocalInput; use crate::executor::exchange::input::Input; use crate::executor::ActorContext; @@ -220,6 +219,7 @@ mod tests { use futures::{pin_mut, FutureExt}; use risingwave_common::array::StreamChunk; + use risingwave_common::catalog::Schema; use risingwave_pb::stream_plan::update_mutation::MergeUpdate; use super::*; diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index 16eafd479a5a2..7c5d66af19b8d 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -19,14 +19,14 @@ use risingwave_common::array::{ Array, ArrayBuilder, ArrayRef, Op, SerialArrayBuilder, StreamChunk, }; use risingwave_common::buffer::Bitmap; -use risingwave_common::catalog::Schema; + use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::types::Serial; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::row_id::RowIdGenerator; use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, PkIndicesRef, + expect_first_barrier, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, }; use crate::executor::{Message, StreamExecutorError}; diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index f474c2d607d4f..29f031a248ece 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -15,13 +15,13 @@ use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::Op; -use risingwave_common::catalog::Schema; + use risingwave_storage::StateStore; use super::sort_buffer::SortBuffer; use super::{ expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorError, Watermark, + ExecutorInfo, Message, StreamExecutorError, Watermark, }; use crate::common::table::state_table::StateTable; use crate::common::StreamChunkBuilder; diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs index ec80272863bd8..4b1de233e2545 100644 --- a/src/stream/src/executor/source/fetch_executor.rs +++ b/src/stream/src/executor/source/fetch_executor.rs @@ -22,7 +22,7 @@ use futures::stream::{self, StreamExt}; use futures::{pin_mut, TryStreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; -use risingwave_common::catalog::{ColumnId, Schema, TableId}; +use risingwave_common::catalog::{ColumnId, TableId}; use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{ScalarRef, ScalarRefImpl}; diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index f12e926fde55c..1ddbdb2e5494a 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -22,7 +22,7 @@ use anyhow::anyhow; use either::Either; use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; -use risingwave_common::catalog::Schema; + use risingwave_common::system_param::local_manager::SystemParamsReaderRef; use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_connector::source::reader::desc::{FsSourceDesc, SourceDescBuilder}; diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs index 8f74703cc896d..92153d5c9666f 100644 --- a/src/stream/src/executor/source/list_executor.rs +++ b/src/stream/src/executor/source/list_executor.rs @@ -20,7 +20,7 @@ use either::Either; use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::Op; -use risingwave_common::catalog::Schema; + use risingwave_common::system_param::local_manager::SystemParamsReaderRef; use risingwave_connector::source::reader::desc::{SourceDesc, SourceDescBuilder}; use risingwave_connector::source::SourceCtrlOpts; diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index f4200888d41e3..bb4a6ca4958de 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -16,7 +16,7 @@ use futures::StreamExt; use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::aggregate::{ build_retractable, AggCall, AggregateState, BoxedAggregateFunction, diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs index a48f9c357b186..2f706b3247f11 100644 --- a/src/stream/src/executor/subscription.rs +++ b/src/stream/src/executor/subscription.rs @@ -16,7 +16,7 @@ use core::time::Duration; use futures::prelude::stream::StreamExt; use futures_async_stream::try_stream; -use risingwave_common::catalog::Schema; + use risingwave_common::types::Timestamptz; use risingwave_common::util::epoch::Epoch; use risingwave_storage::store::LocalStateStore; @@ -24,7 +24,7 @@ use tokio::time::Instant; use super::{ expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorError, StreamExecutorResult, + ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, }; use crate::common::log_store_impl::kv_log_store::ReaderTruncationOffsetType; use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter; diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs index 6576b9e6654ac..3dbcea611a208 100644 --- a/src/stream/src/executor/temporal_join.rs +++ b/src/stream/src/executor/temporal_join.rs @@ -26,7 +26,7 @@ use futures_async_stream::try_stream; use local_stats_alloc::{SharedStatsAlloc, StatsAlloc}; use lru::DefaultHasher; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_common::estimate_size::{EstimateSize, KvSize}; use risingwave_common::hash::{HashKey, NullBitmap}; use risingwave_common::row::{OwnedRow, Row, RowExt}; diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index 0850cd004b2d0..dca30b0f6104d 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -31,7 +31,7 @@ use super::CacheKey; use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, Watermark, + ExecutorInfo, Message, Watermark, }; pub trait TopNExecutorBase: Send + 'static { @@ -82,7 +82,7 @@ where E: TopNExecutorBase, { fn info(&self) -> &ExecutorInfo { - &self.inner.info() + self.inner.info() } fn execute(self: Box) -> BoxedMessageStream { diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index 1b20b1e8f0a49..76fd0d698038c 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -20,7 +20,7 @@ use futures::stream::{FusedStream, FuturesUnordered}; use futures::StreamExt; use futures_async_stream::try_stream; use pin_project::pin_project; -use risingwave_common::catalog::Schema; + use super::watermark::BufferedWatermarks; use super::*; diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs index 859eee061b8c2..eae687da4dabb 100644 --- a/src/stream/src/executor/utils.rs +++ b/src/stream/src/executor/utils.rs @@ -13,11 +13,11 @@ // limitations under the License. use futures::StreamExt; -use risingwave_common::catalog::Schema; + use risingwave_common::metrics::LabelGuardedIntCounter; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{BoxedMessageStream, Executor, ExecutorInfo, PkIndicesRef}; +use crate::executor::{BoxedMessageStream, Executor, ExecutorInfo}; use crate::task::{ActorId, FragmentId}; #[derive(Default)] diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index b918a36e1d89f..0c893c423431c 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -18,14 +18,14 @@ use await_tree::InstrumentAwait; use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::{DataChunk, Op, StreamChunk}; -use risingwave_common::catalog::Schema; + use risingwave_common::ensure; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; use tokio::sync::mpsc::UnboundedReceiver; use super::{ - ActorContextRef, Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, PkIndicesRef, + ActorContextRef, Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, StreamExecutorError, }; use crate::task::CreateMviewProgress; diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index adb82c4b3e4c7..3acbafe509dd2 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -15,7 +15,7 @@ use std::sync::Arc; use futures::StreamExt; -use risingwave_common::catalog::Schema; + use super::*; @@ -90,7 +90,7 @@ impl WrapperExecutor { impl Executor for WrapperExecutor { fn info(&self) -> &ExecutorInfo { - &self.input.info() + self.input.info() } fn execute(self: Box) -> BoxedMessageStream { From 56b54f6f5c286d40e7b1a380bbe8730762eded20 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Fri, 2 Feb 2024 13:38:37 +0800 Subject: [PATCH 07/13] rename `Executor` trait to `Execute` Signed-off-by: Richard Chien --- src/compute/tests/cdc_tests.rs | 4 +-- src/compute/tests/integration_tests.rs | 2 +- src/stream/src/executor/agg_common.rs | 4 +-- .../executor/backfill/arrangement_backfill.rs | 5 ++-- .../src/executor/backfill/cdc/cdc_backfill.rs | 6 ++--- .../executor/backfill/no_shuffle_backfill.rs | 5 ++-- src/stream/src/executor/barrier_recv.rs | 4 +-- src/stream/src/executor/batch_query.rs | 5 ++-- src/stream/src/executor/chain.rs | 7 +++-- .../src/executor/dedup/append_only_dedup.rs | 5 ++-- src/stream/src/executor/dml.rs | 4 +-- src/stream/src/executor/dynamic_filter.rs | 7 ++--- src/stream/src/executor/expand.rs | 7 +++-- src/stream/src/executor/filter.rs | 5 ++-- src/stream/src/executor/flow_control.rs | 4 +-- src/stream/src/executor/hash_agg.rs | 11 +++----- src/stream/src/executor/hash_join.rs | 5 ++-- src/stream/src/executor/hop_window.rs | 8 +++--- src/stream/src/executor/integration_tests.rs | 2 +- src/stream/src/executor/lookup.rs | 9 +++---- src/stream/src/executor/lookup/impl_.rs | 6 ++--- src/stream/src/executor/lookup/sides.rs | 10 +++---- src/stream/src/executor/lookup/tests.rs | 8 +++--- src/stream/src/executor/lookup_union.rs | 3 +-- src/stream/src/executor/merge.rs | 5 ++-- src/stream/src/executor/mod.rs | 4 +-- src/stream/src/executor/mview/materialize.rs | 6 ++--- src/stream/src/executor/no_op.rs | 8 ++---- src/stream/src/executor/now.rs | 9 +++---- src/stream/src/executor/over_window/eowc.rs | 7 +++-- .../src/executor/over_window/general.rs | 6 ++--- src/stream/src/executor/project.rs | 5 ++-- src/stream/src/executor/project_set.rs | 8 +++--- src/stream/src/executor/rearranged_chain.rs | 5 ++-- src/stream/src/executor/receiver.rs | 6 ++--- src/stream/src/executor/row_id_gen.rs | 9 +++---- src/stream/src/executor/simple_agg.rs | 4 +-- src/stream/src/executor/sink.rs | 4 +-- src/stream/src/executor/sort.rs | 7 +++-- .../src/executor/source/fetch_executor.rs | 2 +- .../src/executor/source/fs_source_executor.rs | 3 +-- .../src/executor/source/list_executor.rs | 3 +-- .../src/executor/source/source_executor.rs | 2 +- .../src/executor/stateless_simple_agg.rs | 9 +++---- src/stream/src/executor/subscription.rs | 5 ++-- src/stream/src/executor/subtask.rs | 4 +-- src/stream/src/executor/temporal_join.rs | 7 +++-- src/stream/src/executor/test_utils.rs | 22 ++++++++-------- src/stream/src/executor/top_n/group_top_n.rs | 10 +++---- .../executor/top_n/group_top_n_appendonly.rs | 4 +-- .../src/executor/top_n/top_n_appendonly.rs | 10 +++---- src/stream/src/executor/top_n/top_n_plain.rs | 26 +++++++++---------- src/stream/src/executor/top_n/utils.rs | 4 +-- src/stream/src/executor/union.rs | 3 +-- src/stream/src/executor/utils.rs | 5 ++-- src/stream/src/executor/values.rs | 7 +++-- src/stream/src/executor/watermark_filter.rs | 6 ++--- src/stream/src/executor/wrapper.rs | 3 +-- .../src/executor/wrapper/epoch_check.rs | 2 +- .../src/executor/wrapper/schema_check.rs | 2 +- .../src/executor/wrapper/update_check.rs | 2 +- src/stream/src/from_proto/eowc_over_window.rs | 4 +-- src/stream/src/from_proto/hash_join.rs | 4 +-- src/stream/src/from_proto/mod.rs | 2 +- src/stream/src/from_proto/no_op.rs | 2 +- src/stream/src/from_proto/over_window.rs | 2 +- src/stream/src/from_proto/source/fs_fetch.rs | 2 +- 67 files changed, 171 insertions(+), 215 deletions(-) diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs index 897019bc75925..e26770fbb2c08 100644 --- a/src/compute/tests/cdc_tests.rs +++ b/src/compute/tests/cdc_tests.rs @@ -45,7 +45,7 @@ use risingwave_stream::executor::monitor::StreamingMetrics; use risingwave_stream::executor::test_utils::MockSource; use risingwave_stream::executor::{ expect_first_barrier, ActorContext, AddMutation, Barrier, BoxedExecutor as StreamBoxedExecutor, - BoxedMessageStream, CdcBackfillExecutor, Executor, ExecutorInfo, ExternalStorageTable, + BoxedMessageStream, CdcBackfillExecutor, Execute, ExecutorInfo, ExternalStorageTable, MaterializeExecutor, Message, Mutation, PkIndices, PkIndicesRef, StreamExecutorError, }; @@ -129,7 +129,7 @@ impl MockOffsetGenExecutor { } } -impl Executor for MockOffsetGenExecutor { +impl Execute for MockOffsetGenExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/compute/tests/integration_tests.rs b/src/compute/tests/integration_tests.rs index d9e9e608abc92..f75b082ef79ad 100644 --- a/src/compute/tests/integration_tests.rs +++ b/src/compute/tests/integration_tests.rs @@ -57,7 +57,7 @@ use risingwave_stream::executor::monitor::StreamingMetrics; use risingwave_stream::executor::row_id_gen::RowIdGenExecutor; use risingwave_stream::executor::source_executor::SourceExecutor; use risingwave_stream::executor::{ - ActorContext, Barrier, Executor, ExecutorInfo, MaterializeExecutor, Message, PkIndices, + ActorContext, Barrier, Execute, ExecutorInfo, MaterializeExecutor, Message, PkIndices, }; use tokio::sync::mpsc::unbounded_channel; diff --git a/src/stream/src/executor/agg_common.rs b/src/stream/src/executor/agg_common.rs index 603cb7f244f87..fa7a3f9f93a9b 100644 --- a/src/stream/src/executor/agg_common.rs +++ b/src/stream/src/executor/agg_common.rs @@ -19,7 +19,7 @@ use risingwave_pb::stream_plan::PbAggNodeVersion; use risingwave_storage::StateStore; use super::aggregation::AggStateStorage; -use super::{Executor, ExecutorInfo}; +use super::{Execute, ExecutorInfo}; use crate::common::table::state_table::StateTable; use crate::executor::ActorContextRef; use crate::task::AtomicU64Ref; @@ -29,7 +29,7 @@ pub struct AggExecutorArgs { pub version: PbAggNodeVersion, // basic - pub input: Box, + pub input: Box, pub actor_ctx: ActorContextRef, pub info: ExecutorInfo, diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 40bf19ea32cf8..0c979842212fa 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -22,7 +22,6 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::bail; - use risingwave_common::hash::{VirtualNode, VnodeBitmapExt}; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; use risingwave_common::util::iter_util::ZipEqDebug; @@ -40,7 +39,7 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, + expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, StreamExecutorError, }; use crate::task::{ActorId, CreateMviewProgress}; @@ -631,7 +630,7 @@ where } } -impl Executor for ArrangementBackfillExecutor +impl Execute for ArrangementBackfillExecutor where S: StateStore, SD: ValueRowSerde, diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index c2aad5d7e460b..e3c3d08e86b2e 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -45,7 +45,7 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, + expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, }; use crate::task::CreateMviewProgress; @@ -589,7 +589,7 @@ fn get_rw_columns(schema: &Schema) -> Vec { .collect_vec() } -impl Executor for CdcBackfillExecutor { +impl Execute for CdcBackfillExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -611,7 +611,7 @@ mod tests { use crate::executor::backfill::cdc::cdc_backfill::transform_upstream; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; + use crate::executor::Execute; #[tokio::test] async fn test_transform_upstream_chunk() { diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index 86fb1766b8cca..d40568c22a2a8 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -20,7 +20,6 @@ use futures::stream::select_with_strategy; use futures::{stream, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{DataChunk, Op, StreamChunk}; - use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::Datum; @@ -39,7 +38,7 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, + expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, Mutation, StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; @@ -739,7 +738,7 @@ where } } -impl Executor for BackfillExecutor +impl Execute for BackfillExecutor where S: StateStore, { diff --git a/src/stream/src/executor/barrier_recv.rs b/src/stream/src/executor/barrier_recv.rs index bb30850f69c69..4fce4cc34d545 100644 --- a/src/stream/src/executor/barrier_recv.rs +++ b/src/stream/src/executor/barrier_recv.rs @@ -18,7 +18,7 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ - ActorContext, ActorContextRef, Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, + ActorContext, ActorContextRef, Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, PkIndices, StreamExecutorError, }; @@ -58,7 +58,7 @@ impl BarrierRecvExecutor { } } -impl Executor for BarrierRecvExecutor { +impl Execute for BarrierRecvExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs index be476e20e670c..5fc2607179d52 100644 --- a/src/stream/src/executor/batch_query.rs +++ b/src/stream/src/executor/batch_query.rs @@ -16,7 +16,6 @@ use await_tree::InstrumentAwait; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{Op, StreamChunk}; - use risingwave_hummock_sdk::HummockReadEpoch; use risingwave_storage::store::PrefetchOptions; use risingwave_storage::table::batch_table::storage_table::StorageTable; @@ -24,7 +23,7 @@ use risingwave_storage::table::collect_data_chunk; use risingwave_storage::StateStore; use super::error::StreamExecutorError; -use super::{Executor, ExecutorInfo, Message}; +use super::{Execute, ExecutorInfo, Message}; use crate::executor::BoxedMessageStream; pub struct BatchQueryExecutor { @@ -73,7 +72,7 @@ where } } -impl Executor for BatchQueryExecutor +impl Execute for BatchQueryExecutor where S: StateStore, { diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index 5a08a9e56d185..ca40405302fb8 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -15,9 +15,8 @@ use futures::StreamExt; use futures_async_stream::try_stream; - use super::error::StreamExecutorError; -use super::{expect_first_barrier, BoxedExecutor, Executor, ExecutorInfo, Message}; +use super::{expect_first_barrier, BoxedExecutor, Execute, ExecutorInfo, Message}; use crate::task::{ActorId, CreateMviewProgress}; /// [`ChainExecutor`] is an executor that enables synchronization between the existing stream and @@ -104,7 +103,7 @@ impl ChainExecutor { } } -impl Executor for ChainExecutor { +impl Execute for ChainExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -128,7 +127,7 @@ mod test { use super::ChainExecutor; use crate::executor::test_utils::MockSource; use crate::executor::{ - AddMutation, Barrier, Executor, ExecutorInfo, Message, Mutation, PkIndices, + AddMutation, Barrier, Execute, ExecutorInfo, Message, Mutation, PkIndices, }; use crate::task::{CreateMviewProgress, LocalBarrierManager}; diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index 78294d0f370ca..3ead68eeea689 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -19,7 +19,6 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::BitmapBuilder; - use risingwave_common::row::{OwnedRow, Row, RowExt}; use risingwave_storage::StateStore; @@ -29,7 +28,7 @@ use crate::common::table::state_table::StateTable; use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, + expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, StreamExecutorResult, }; use crate::task::AtomicU64Ref; @@ -195,7 +194,7 @@ impl AppendOnlyDedupExecutor { } } -impl Executor for AppendOnlyDedupExecutor { +impl Execute for AppendOnlyDedupExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index 35f14e5284703..96db02a08b09d 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -26,7 +26,7 @@ use risingwave_dml::dml_manager::DmlManagerRef; use super::error::StreamExecutorError; use super::{ - expect_first_barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, + expect_first_barrier, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, Mutation, }; use crate::common::StreamChunkBuilder; @@ -277,7 +277,7 @@ impl DmlExecutor { } } -impl Executor for DmlExecutor { +impl Execute for DmlExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index 4c68125eb04aa..940b29ef3e1d6 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -20,7 +20,6 @@ use futures_async_stream::try_stream; use risingwave_common::array::{Array, ArrayImpl, Op, StreamChunk}; use risingwave_common::bail; use risingwave_common::buffer::{Bitmap, BitmapBuilder}; - use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{self, once, OwnedRow, OwnedRow as RowData, Row}; use risingwave_common::types::{DataType, Datum, DefaultOrd, ScalarImpl, ToDatumRef, ToOwnedDatum}; @@ -38,9 +37,7 @@ use risingwave_storage::StateStore; use super::barrier_align::*; use super::error::StreamExecutorError; use super::monitor::StreamingMetrics; -use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, -}; +use super::{ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message}; use crate::common::table::state_table::{StateTable, WatermarkCacheParameterizedStateTable}; use crate::common::StreamChunkBuilder; use crate::executor::expect_first_barrier_from_aligned_stream; @@ -490,7 +487,7 @@ impl DynamicFilterExecutor Executor +impl Execute for DynamicFilterExecutor { fn info(&self) -> &ExecutorInfo { diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index dd36c559bffa5..3af392ca3164b 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -18,7 +18,6 @@ use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::{Array, I64Array}; - use super::error::StreamExecutorError; use super::*; @@ -31,7 +30,7 @@ pub struct ExpandExecutor { impl ExpandExecutor { pub fn new( info: ExecutorInfo, - input: Box, + input: Box, column_subsets: Vec>, ) -> Self { Self { @@ -73,7 +72,7 @@ impl Debug for ExpandExecutor { } } -impl Executor for ExpandExecutor { +impl Execute for ExpandExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -92,7 +91,7 @@ mod tests { use super::ExpandExecutor; use crate::executor::test_utils::MockSource; - use crate::executor::{Executor, ExecutorInfo, PkIndices}; + use crate::executor::{Execute, ExecutorInfo, PkIndices}; #[tokio::test] async fn test_expand() { diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index 33a92aa4216e3..a6bce873d7d96 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use risingwave_common::array::{Array, ArrayImpl, Op, StreamChunk}; use risingwave_common::buffer::BitmapBuilder; - use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; @@ -41,7 +40,7 @@ impl FilterExecutor { pub fn new( ctx: ActorContextRef, info: ExecutorInfo, - input: Box, + input: Box, expr: NonStrictExpression, ) -> Self { Self { @@ -135,7 +134,7 @@ impl Debug for FilterExecutor { } } -impl Executor for FilterExecutor { +impl Execute for FilterExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs index 022796b251ebb..1719611507c35 100644 --- a/src/stream/src/executor/flow_control.rs +++ b/src/stream/src/executor/flow_control.rs @@ -38,7 +38,7 @@ pub struct FlowControlExecutor { impl FlowControlExecutor { pub fn new( - input: Box, + input: Box, actor_ctx: ActorContextRef, rate_limit: Option, ) -> Self { @@ -130,7 +130,7 @@ impl Debug for FlowControlExecutor { } } -impl Executor for FlowControlExecutor { +impl Execute for FlowControlExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index 1bf50bb13e1b6..bea9a8f696932 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -38,10 +38,7 @@ use super::aggregation::{ OnlyOutputIfHasInput, }; use super::sort_buffer::SortBuffer; -use super::{ - expect_first_barrier, ActorContextRef, ExecutorInfo, StreamExecutorResult, - Watermark, -}; +use super::{expect_first_barrier, ActorContextRef, ExecutorInfo, StreamExecutorResult, Watermark}; use crate::cache::{cache_may_stale, new_with_hasher, ManagedLruCache}; use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; @@ -49,7 +46,7 @@ use crate::common::StreamChunkBuilder; use crate::error::StreamResult; use crate::executor::aggregation::AggGroup as GenericAggGroup; use crate::executor::error::StreamExecutorError; -use crate::executor::{BoxedMessageStream, Executor, Message}; +use crate::executor::{BoxedMessageStream, Execute, Message}; use crate::task::AtomicU64Ref; type AggGroup = GenericAggGroup; @@ -74,7 +71,7 @@ type AggGroupCache = ManagedLruCache>, Precompu /// all modifications will be flushed to the storage backend. Meanwhile, the executor will go /// through `group_change_set`, and produce a stream chunk based on the state changes. pub struct HashAggExecutor { - input: Box, + input: Box, inner: ExecutorInner, } @@ -193,7 +190,7 @@ impl ExecutionStats { } } -impl Executor for HashAggExecutor { +impl Execute for HashAggExecutor { fn info(&self) -> &ExecutorInfo { &self.inner.info } diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index 998db81cc1b12..1c38b62eed6ff 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -22,7 +22,6 @@ use futures_async_stream::try_stream; use itertools::Itertools; use multimap::MultiMap; use risingwave_common::array::{Op, RowRef, StreamChunk}; - use risingwave_common::hash::{HashKey, NullBitmap}; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, DefaultOrd, ToOwnedDatum}; @@ -40,7 +39,7 @@ use super::managed_state::join::*; use super::monitor::StreamingMetrics; use super::watermark::*; use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, Watermark, + ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, Watermark, }; use crate::common::table::state_table::StateTable; use crate::common::JoinStreamChunkBuilder; @@ -276,7 +275,7 @@ impl std::fmt::Debug } } -impl Executor for HashJoinExecutor { +impl Execute for HashJoinExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs index 43a05fe4ff04b..a192156205f85 100644 --- a/src/stream/src/executor/hop_window.rs +++ b/src/stream/src/executor/hop_window.rs @@ -23,7 +23,7 @@ use risingwave_expr::expr::NonStrictExpression; use risingwave_expr::ExprError; use super::error::StreamExecutorError; -use super::{ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message}; +use super::{ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message}; use crate::common::StreamChunkBuilder; pub struct HopWindowExecutor { @@ -68,7 +68,7 @@ impl HopWindowExecutor { } } -impl Executor for HopWindowExecutor { +impl Execute for HopWindowExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -246,11 +246,11 @@ mod tests { use risingwave_expr::expr::NonStrictExpression; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Executor, ExecutorInfo, StreamChunk}; + use crate::executor::{ActorContext, Execute, ExecutorInfo, StreamChunk}; const CHUNK_SIZE: usize = 256; - fn create_executor(output_indices: Vec) -> Box { + fn create_executor(output_indices: Vec) -> Box { let field1 = Field::unnamed(DataType::Int64); let field2 = Field::unnamed(DataType::Int64); let field3 = Field::with_name(DataType::Timestamp, "created_at"); diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs index 0759f4bd2d813..e3652cce0398f 100644 --- a/src/stream/src/executor/integration_tests.rs +++ b/src/stream/src/executor/integration_tests.rs @@ -35,7 +35,7 @@ use crate::executor::receiver::ReceiverExecutor; use crate::executor::test_utils::agg_executor::{ generate_agg_schema, new_boxed_simple_agg_executor, }; -use crate::executor::{Executor, MergeExecutor, ProjectExecutor, StatelessSimpleAggExecutor}; +use crate::executor::{Execute, MergeExecutor, ProjectExecutor, StatelessSimpleAggExecutor}; use crate::task::{LocalBarrierManager, SharedContext}; /// This test creates a merger-dispatcher pair, and run a sum. Each chunk diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs index c8cb8848ba396..185d3fb5a6582 100644 --- a/src/stream/src/executor/lookup.rs +++ b/src/stream/src/executor/lookup.rs @@ -14,11 +14,10 @@ use async_trait::async_trait; use futures::StreamExt; - use risingwave_common::types::DataType; use risingwave_storage::StateStore; -use crate::executor::{Barrier, BoxedMessageStream, Executor}; +use crate::executor::{Barrier, BoxedMessageStream, Execute}; mod cache; mod sides; @@ -54,10 +53,10 @@ pub struct LookupExecutor { stream: StreamJoinSide, /// The executor for arrangement. - arrangement_executor: Option>, + arrangement_executor: Option>, /// The executor for stream. - stream_executor: Option>, + stream_executor: Option>, /// The last received barrier. last_barrier: Option, @@ -83,7 +82,7 @@ pub struct LookupExecutor { } #[async_trait] -impl Executor for LookupExecutor { +impl Execute for LookupExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/lookup/impl_.rs b/src/stream/src/executor/lookup/impl_.rs index 8b86cfc602f08..2707381e8ef6e 100644 --- a/src/stream/src/executor/lookup/impl_.rs +++ b/src/stream/src/executor/lookup/impl_.rs @@ -36,7 +36,7 @@ use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; use crate::executor::lookup::cache::LookupCache; use crate::executor::lookup::sides::{ArrangeJoinSide, ArrangeMessage, StreamJoinSide}; use crate::executor::lookup::LookupExecutor; -use crate::executor::{ActorContextRef, Barrier, Executor, ExecutorInfo, Message}; +use crate::executor::{ActorContextRef, Barrier, Execute, ExecutorInfo, Message}; use crate::task::AtomicU64Ref; /// Parameters for [`LookupExecutor`]. @@ -46,11 +46,11 @@ pub struct LookupExecutorParams { /// The side for arrangement. Currently, it should be a /// `MaterializeExecutor`. - pub arrangement: Box, + pub arrangement: Box, /// The side for stream. It can be any stream, but it will generally be a /// `MaterializeExecutor`. - pub stream: Box, + pub stream: Box, /// Should be the same as [`ColumnDesc`] in the arrangement. /// diff --git a/src/stream/src/executor/lookup/sides.rs b/src/stream/src/executor/lookup/sides.rs index 26bfdd39ec401..abc972c76b8b8 100644 --- a/src/stream/src/executor/lookup/sides.rs +++ b/src/stream/src/executor/lookup/sides.rs @@ -28,7 +28,7 @@ use risingwave_storage::table::batch_table::storage_table::StorageTable; use risingwave_storage::StateStore; use crate::executor::error::StreamExecutorError; -use crate::executor::{Barrier, BoxedMessageStream, Executor, Message, MessageStream}; +use crate::executor::{Barrier, BoxedMessageStream, Execute, Message, MessageStream}; /// Join side of Lookup Executor's stream pub(crate) struct StreamJoinSide { @@ -212,8 +212,8 @@ pub async fn align_barrier(mut left: BoxedMessageStream, mut right: BoxedMessage /// * `[Msg`] Arrangement (batch) #[try_stream(ok = ArrangeMessage, error = StreamExecutorError)] pub async fn stream_lookup_arrange_prev_epoch( - stream: Box, - arrangement: Box, + stream: Box, + arrangement: Box, ) { let mut input = pin!(align_barrier(stream.execute(), arrangement.execute())); let mut arrange_buf = vec![]; @@ -296,8 +296,8 @@ pub async fn stream_lookup_arrange_prev_epoch( /// * Barrier (prev = `[2`], current = `[3`]) #[try_stream(ok = ArrangeMessage, error = StreamExecutorError)] pub async fn stream_lookup_arrange_this_epoch( - stream: Box, - arrangement: Box, + stream: Box, + arrangement: Box, ) { let mut input = pin!(align_barrier(stream.execute(), arrangement.execute())); let mut stream_buf = vec![]; diff --git a/src/stream/src/executor/lookup/tests.rs b/src/stream/src/executor/lookup/tests.rs index 7d152e8a5f4e7..9f55bd30fb479 100644 --- a/src/stream/src/executor/lookup/tests.rs +++ b/src/stream/src/executor/lookup/tests.rs @@ -30,8 +30,8 @@ use crate::executor::lookup::impl_::LookupExecutorParams; use crate::executor::lookup::LookupExecutor; use crate::executor::test_utils::*; use crate::executor::{ - ActorContext, Barrier, BoxedMessageStream, Executor, ExecutorInfo, MaterializeExecutor, - Message, PkIndices, + ActorContext, Barrier, BoxedMessageStream, Execute, ExecutorInfo, MaterializeExecutor, Message, + PkIndices, }; fn arrangement_col_descs() -> Vec { @@ -71,7 +71,7 @@ fn arrangement_col_arrange_rules_join_key() -> Vec { async fn create_arrangement( table_id: TableId, memory_state_store: MemoryStateStore, -) -> Box { +) -> Box { // Two columns of int32 type, the second column is arrange key. let columns = arrangement_col_descs(); @@ -139,7 +139,7 @@ async fn create_arrangement( /// | b | | | 2 -> 3 | /// | - | 6 | 1 | 3 | /// | b | | | 3 -> 4 | -fn create_source() -> Box { +fn create_source() -> Box { let columns = vec![ ColumnDesc::new_atomic(DataType::Int64, "join_column", 1), ColumnDesc::new_atomic(DataType::Int64, "rowid_column", 2), diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index 0451da2b8ff3c..5f2c203f6aac2 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -19,7 +19,6 @@ use futures::{FutureExt, SinkExt, StreamExt}; use futures_async_stream::try_stream; use itertools::Itertools; - use super::error::StreamExecutorError; use super::*; use crate::executor::{BoxedMessageStream, ExecutorInfo}; @@ -54,7 +53,7 @@ impl LookupUnionExecutor { } #[async_trait] -impl Executor for LookupUnionExecutor { +impl Execute for LookupUnionExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index 1439fa64a0269..e2105aabc236c 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -20,7 +20,6 @@ use anyhow::Context as _; use futures::stream::{FusedStream, FuturesUnordered, StreamFuture}; use futures::{pin_mut, Stream, StreamExt}; use futures_async_stream::try_stream; - use tokio::time::Instant; use super::error::StreamExecutorError; @@ -245,7 +244,7 @@ impl MergeExecutor { } } -impl Executor for MergeExecutor { +impl Execute for MergeExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -456,7 +455,7 @@ mod tests { use super::*; use crate::executor::exchange::input::RemoteInput; use crate::executor::exchange::permit::channel_for_test; - use crate::executor::{Barrier, Executor, Mutation}; + use crate::executor::{Barrier, Execute, Mutation}; use crate::task::test_utils::helper_make_local_actor; fn build_test_chunk(epoch: u64) -> StreamChunk { diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index 6ea6272bcbfe4..ef4943b399d26 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -153,7 +153,7 @@ pub use wrapper::WrapperExecutor; use self::barrier_align::AlignedMessageStream; -pub type BoxedExecutor = Box; +pub type BoxedExecutor = Box; pub type MessageStreamItem = StreamExecutorResult; pub type BoxedMessageStream = BoxStream<'static, MessageStreamItem>; @@ -178,7 +178,7 @@ pub struct ExecutorInfo { } /// `Executor` supports handling of control messages. -pub trait Executor: Send + 'static { +pub trait Execute: Send + 'static { fn info(&self) -> &ExecutorInfo; fn schema(&self) -> &Schema { diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index 77ccf66b05dba..e354508fe3307 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -43,8 +43,8 @@ use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ expect_first_barrier, ActorContext, ActorContextRef, AddMutation, BoxedExecutor, - BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, - StreamExecutorResult, UpdateMutation, + BoxedMessageStream, Execute, ExecutorInfo, Message, Mutation, StreamExecutorResult, + UpdateMutation, }; use crate::task::{ActorId, AtomicU64Ref}; @@ -429,7 +429,7 @@ impl MaterializeBuffer { self.buffer } } -impl Executor for MaterializeExecutor { +impl Execute for MaterializeExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/no_op.rs b/src/stream/src/executor/no_op.rs index 4f4ac666aa732..fcc024f6b9a2a 100644 --- a/src/stream/src/executor/no_op.rs +++ b/src/stream/src/executor/no_op.rs @@ -12,11 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. - - -use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, -}; +use super::{ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo}; /// No-op executor directly forwards the input stream. Currently used to break the multiple edges in /// the fragment graph. @@ -36,7 +32,7 @@ impl NoOpExecutor { } } -impl Executor for NoOpExecutor { +impl Execute for NoOpExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index 3bd8809158a32..0c00b25d4ca09 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -18,7 +18,6 @@ use std::ops::Bound::Unbounded; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{Op, StreamChunk}; - use risingwave_common::row::{self, OwnedRow}; use risingwave_common::types::{DataType, Datum}; use risingwave_storage::StateStore; @@ -26,8 +25,8 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ - Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, - StreamExecutorError, Watermark, + Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, Mutation, StreamExecutorError, + Watermark, }; use crate::common::table::state_table::StateTable; @@ -158,7 +157,7 @@ impl NowExecutor { } } -impl Executor for NowExecutor { +impl Execute for NowExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -181,7 +180,7 @@ mod tests { use crate::common::table::state_table::StateTable; use crate::executor::test_utils::StreamExecutorTestExt; use crate::executor::{ - Barrier, BoxedMessageStream, Executor, ExecutorInfo, Mutation, StreamExecutorResult, + Barrier, BoxedMessageStream, Execute, ExecutorInfo, Mutation, StreamExecutorResult, Watermark, }; diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 42a1f41d6f4e3..099bbade8a66c 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -20,7 +20,6 @@ use futures_async_stream::{for_await, try_stream}; use itertools::Itertools; use risingwave_common::array::stream_record::Record; use risingwave_common::array::{ArrayRef, Op, StreamChunk}; - use risingwave_common::estimate_size::collections::EstimatedVecDeque; use risingwave_common::estimate_size::EstimateSize; use risingwave_common::row::{OwnedRow, Row, RowExt}; @@ -39,7 +38,7 @@ use crate::cache::{new_unbounded, ManagedLruCache}; use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, + expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, }; use crate::task::AtomicU64Ref; @@ -95,7 +94,7 @@ type PartitionCache = ManagedLruCache; // TODO(rc): us /// - `WindowState` should output agg result for `curr output row`. /// - Recover: iterate through state table, push rows to `WindowState`, ignore ready windows. pub struct EowcOverWindowExecutor { - input: Box, + input: Box, inner: ExecutorInner, } @@ -117,7 +116,7 @@ struct ExecutionVars { _phantom: PhantomData, } -impl Executor for EowcOverWindowExecutor { +impl Execute for EowcOverWindowExecutor { fn info(&self) -> &ExecutorInfo { &self.inner.info } diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index 3519e9d348471..dcc59fd0ef166 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -45,7 +45,7 @@ use crate::common::table::state_table::StateTable; use crate::common::StreamChunkBuilder; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, + expect_first_barrier, ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, }; use crate::task::AtomicU64Ref; @@ -56,7 +56,7 @@ use crate::task::AtomicU64Ref; /// - State table schema = output schema, state table pk = `partition key | order key | input pk`. /// - Output schema = input schema + window function results. pub struct OverWindowExecutor { - input: Box, + input: Box, inner: ExecutorInner, } @@ -97,7 +97,7 @@ struct ExecutionStats { cache_lookup: u64, } -impl Executor for OverWindowExecutor { +impl Execute for OverWindowExecutor { fn info(&self) -> &ExecutorInfo { &self.inner.info } diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index 1a995d9c0ea2f..8404bc5b9013d 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -16,7 +16,6 @@ use std::fmt::{Debug, Formatter}; use multimap::MultiMap; use risingwave_common::array::StreamChunk; - use risingwave_common::row::{Row, RowExt}; use risingwave_common::types::ToOwnedDatum; use risingwave_common::util::iter_util::ZipEqFast; @@ -56,7 +55,7 @@ impl ProjectExecutor { pub fn new( ctx: ActorContextRef, info: ExecutorInfo, - input: Box, + input: Box, exprs: Vec, watermark_derivations: MultiMap, nondecreasing_expr_indices: Vec, @@ -86,7 +85,7 @@ impl Debug for ProjectExecutor { } } -impl Executor for ProjectExecutor { +impl Execute for ProjectExecutor { fn info(&self) -> &ExecutorInfo { &self.inner.info } diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs index 35bb5c793e6fa..ed327cb97abf0 100644 --- a/src/stream/src/executor/project_set.rs +++ b/src/stream/src/executor/project_set.rs @@ -20,7 +20,6 @@ use futures_async_stream::try_stream; use multimap::MultiMap; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::bail; - use risingwave_common::row::{Row, RowExt}; use risingwave_common::types::{DataType, Datum, DatumRef, ToOwnedDatum}; use risingwave_common::util::iter_util::ZipEqFast; @@ -29,8 +28,7 @@ use risingwave_expr::table_function::ProjectSetSelectItem; use super::error::StreamExecutorError; use super::{ - ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, - StreamExecutorResult, Watermark, + ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message, StreamExecutorResult, Watermark, }; use crate::common::StreamChunkBuilder; @@ -63,7 +61,7 @@ impl ProjectSetExecutor { pub fn new( ctx: ActorContextRef, info: ExecutorInfo, - input: Box, + input: Box, select_list: Vec, chunk_size: usize, watermark_derivations: MultiMap, @@ -90,7 +88,7 @@ impl Debug for ProjectSetExecutor { } } -impl Executor for ProjectSetExecutor { +impl Execute for ProjectSetExecutor { fn info(&self) -> &ExecutorInfo { &self.inner.info } diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs index 0c23bab02e1a1..f56426778c394 100644 --- a/src/stream/src/executor/rearranged_chain.rs +++ b/src/stream/src/executor/rearranged_chain.rs @@ -20,10 +20,9 @@ use futures::{stream, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; - use super::error::StreamExecutorError; use super::{ - expect_first_barrier, Barrier, BoxedExecutor, Executor, ExecutorInfo, Message, MessageStream, + expect_first_barrier, Barrier, BoxedExecutor, Execute, ExecutorInfo, Message, MessageStream, }; use crate::task::{ActorId, CreateMviewProgress}; @@ -288,7 +287,7 @@ impl RearrangedChainExecutor { } } -impl Executor for RearrangedChainExecutor { +impl Execute for RearrangedChainExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index fc84191d2ffc2..513e71ddd5321 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -24,7 +24,7 @@ use super::ActorContextRef; use crate::executor::exchange::input::new_input; use crate::executor::monitor::StreamingMetrics; use crate::executor::utils::ActorInputMetrics; -use crate::executor::{expect_first_barrier, BoxedMessageStream, Executor, ExecutorInfo, Message}; +use crate::executor::{expect_first_barrier, BoxedMessageStream, Execute, ExecutorInfo, Message}; use crate::task::{FragmentId, SharedContext}; /// `ReceiverExecutor` is used along with a channel. After creating a mpsc channel, /// there should be a `ReceiverExecutor` running in the background, so as to push @@ -109,7 +109,7 @@ impl ReceiverExecutor { } } -impl Executor for ReceiverExecutor { +impl Execute for ReceiverExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -223,7 +223,7 @@ mod tests { use risingwave_pb::stream_plan::update_mutation::MergeUpdate; use super::*; - use crate::executor::{ActorContext, Barrier, Executor, Mutation, UpdateMutation}; + use crate::executor::{ActorContext, Barrier, Execute, Mutation, UpdateMutation}; use crate::task::test_utils::helper_make_local_actor; #[tokio::test] diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index 7c5d66af19b8d..dd52aad8cd86d 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -19,15 +19,12 @@ use risingwave_common::array::{ Array, ArrayBuilder, ArrayRef, Op, SerialArrayBuilder, StreamChunk, }; use risingwave_common::buffer::Bitmap; - use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::types::Serial; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::row_id::RowIdGenerator; -use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, -}; +use super::{expect_first_barrier, ActorContextRef, BoxedExecutor, Execute, ExecutorInfo}; use crate::executor::{Message, StreamExecutorError}; /// [`RowIdGenExecutor`] generates row id for data, where the user has not specified a pk. @@ -126,7 +123,7 @@ impl RowIdGenExecutor { } } -impl Executor for RowIdGenExecutor { +impl Execute for RowIdGenExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -146,7 +143,7 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Executor}; + use crate::executor::{ActorContext, Execute}; #[tokio::test] async fn test_row_id_gen_executor() { diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index 65ef8f7be895c..4b3ea132f100f 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -47,7 +47,7 @@ use crate::task::AtomicU64Ref; /// Therefore, we "automatically" implemented a window function inside /// `SimpleAggExecutor`. pub struct SimpleAggExecutor { - input: Box, + input: Box, inner: ExecutorInner, } @@ -111,7 +111,7 @@ struct ExecutionVars { state_changed: bool, } -impl Executor for SimpleAggExecutor { +impl Execute for SimpleAggExecutor { fn info(&self) -> &ExecutorInfo { &self.inner.info } diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index 444b1cfcff979..b2eca12ad71bc 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -34,7 +34,7 @@ use risingwave_connector::sink::{ use thiserror_ext::AsReport; use super::error::{StreamExecutorError, StreamExecutorResult}; -use super::{BoxedExecutor, Executor, ExecutorInfo, Message, PkIndices}; +use super::{BoxedExecutor, Execute, ExecutorInfo, Message, PkIndices}; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedMessageStream, MessageStream, Mutation, }; @@ -402,7 +402,7 @@ impl SinkExecutor { } } -impl Executor for SinkExecutor { +impl Execute for SinkExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index 29f031a248ece..619afb7579e88 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -15,12 +15,11 @@ use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::Op; - use risingwave_storage::StateStore; use super::sort_buffer::SortBuffer; use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, + expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, StreamExecutorError, Watermark, }; use crate::common::table::state_table::StateTable; @@ -56,7 +55,7 @@ struct ExecutionVars { buffer_changed: bool, } -impl Executor for SortExecutor { +impl Execute for SortExecutor { fn info(&self) -> &ExecutorInfo { &self.inner.info } @@ -173,7 +172,7 @@ mod tests { use super::*; use crate::executor::test_utils::{MessageSender, MockSource, StreamExecutorTestExt}; - use crate::executor::{ActorContext, BoxedMessageStream, Executor}; + use crate::executor::{ActorContext, BoxedMessageStream, Execute}; async fn create_executor( sort_column_index: usize, diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs index 4b1de233e2545..e50f882febd80 100644 --- a/src/stream/src/executor/source/fetch_executor.rs +++ b/src/stream/src/executor/source/fetch_executor.rs @@ -344,7 +344,7 @@ impl FsFetchExecutor { } } -impl Executor for FsFetchExecutor { +impl Execute for FsFetchExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index 1ddbdb2e5494a..c1b3b526445c7 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -22,7 +22,6 @@ use anyhow::anyhow; use either::Either; use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; - use risingwave_common::system_param::local_manager::SystemParamsReaderRef; use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_connector::source::reader::desc::{FsSourceDesc, SourceDescBuilder}; @@ -476,7 +475,7 @@ impl FsSourceExecutor { } } -impl Executor for FsSourceExecutor { +impl Execute for FsSourceExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs index 92153d5c9666f..a96b2dc392e50 100644 --- a/src/stream/src/executor/source/list_executor.rs +++ b/src/stream/src/executor/source/list_executor.rs @@ -20,7 +20,6 @@ use either::Either; use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::Op; - use risingwave_common::system_param::local_manager::SystemParamsReaderRef; use risingwave_connector::source::reader::desc::{SourceDesc, SourceDescBuilder}; use risingwave_connector::source::SourceCtrlOpts; @@ -193,7 +192,7 @@ impl FsListExecutor { } } -impl Executor for FsListExecutor { +impl Execute for FsListExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index d6965a3b6884d..22d765c4954e6 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -657,7 +657,7 @@ impl SourceExecutor { } } -impl Executor for SourceExecutor { +impl Execute for SourceExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index bb4a6ca4958de..86862f2f3c41d 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -16,7 +16,6 @@ use futures::StreamExt; use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; - use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::aggregate::{ build_retractable, AggCall, AggregateState, BoxedAggregateFunction, @@ -30,12 +29,12 @@ use crate::error::StreamResult; pub struct StatelessSimpleAggExecutor { _ctx: ActorContextRef, pub(super) info: ExecutorInfo, - pub(super) input: Box, + pub(super) input: Box, pub(super) aggs: Vec, pub(super) agg_calls: Vec, } -impl Executor for StatelessSimpleAggExecutor { +impl Execute for StatelessSimpleAggExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -117,7 +116,7 @@ impl StatelessSimpleAggExecutor { pub fn new( ctx: ActorContextRef, info: ExecutorInfo, - input: Box, + input: Box, agg_calls: Vec, ) -> StreamResult { let aggs = agg_calls.iter().map(build_retractable).try_collect()?; @@ -142,7 +141,7 @@ mod tests { use super::*; use crate::executor::test_utils::agg_executor::generate_agg_schema; use crate::executor::test_utils::MockSource; - use crate::executor::{Executor, StatelessSimpleAggExecutor}; + use crate::executor::{Execute, StatelessSimpleAggExecutor}; #[tokio::test] async fn test_no_chunk() { diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs index 2f706b3247f11..3820d6a5bea42 100644 --- a/src/stream/src/executor/subscription.rs +++ b/src/stream/src/executor/subscription.rs @@ -16,14 +16,13 @@ use core::time::Duration; use futures::prelude::stream::StreamExt; use futures_async_stream::try_stream; - use risingwave_common::types::Timestamptz; use risingwave_common::util::epoch::Epoch; use risingwave_storage::store::LocalStateStore; use tokio::time::Instant; use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, + expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, }; use crate::common::log_store_impl::kv_log_store::ReaderTruncationOffsetType; @@ -115,7 +114,7 @@ impl SubscriptionExecutor { } } } -impl Executor for SubscriptionExecutor { +impl Execute for SubscriptionExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index e313b42d7d434..b7086239ad1da 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -20,7 +20,7 @@ use tokio::sync::mpsc::error::SendError; use tokio_stream::wrappers::ReceiverStream; use super::actor::spawn_blocking_drop_stream; -use super::{BoxedExecutor, Executor, ExecutorInfo, Message, MessageStreamItem}; +use super::{BoxedExecutor, Execute, ExecutorInfo, Message, MessageStreamItem}; use crate::task::ActorId; /// Handle used to drive the subtask. @@ -34,7 +34,7 @@ pub struct SubtaskRxExecutor { rx: mpsc::Receiver, } -impl Executor for SubtaskRxExecutor { +impl Execute for SubtaskRxExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs index 3dbcea611a208..9a74b4fb74a38 100644 --- a/src/stream/src/executor/temporal_join.rs +++ b/src/stream/src/executor/temporal_join.rs @@ -26,7 +26,6 @@ use futures_async_stream::try_stream; use local_stats_alloc::{SharedStatsAlloc, StatsAlloc}; use lru::DefaultHasher; use risingwave_common::array::{Op, StreamChunk}; - use risingwave_common::estimate_size::{EstimateSize, KvSize}; use risingwave_common::hash::{HashKey, NullBitmap}; use risingwave_common::row::{OwnedRow, Row, RowExt}; @@ -40,7 +39,7 @@ use risingwave_storage::table::TableIter; use risingwave_storage::StateStore; use super::{ - Barrier, Executor, ExecutorInfo, Message, MessageStream, StreamExecutorError, + Barrier, Execute, ExecutorInfo, Message, MessageStream, StreamExecutorError, StreamExecutorResult, }; use crate::cache::{cache_may_stale, new_with_hasher_in, ManagedLruCache}; @@ -280,7 +279,7 @@ async fn internal_messages_until_barrier(stream: impl MessageStream, expected_ba // any number of `InternalMessage::Chunk(left_chunk)` and followed by // `InternalMessage::Barrier(right_chunks, barrier)`. #[try_stream(ok = InternalMessage, error = StreamExecutorError)] -async fn align_input(left: Box, right: Box) { +async fn align_input(left: Box, right: Box) { let mut left = pin!(left.execute()); let mut right = pin!(right.execute()); // Keep producing intervals until stream exhaustion or errors. @@ -488,7 +487,7 @@ impl TemporalJoinExecutor } } -impl Executor +impl Execute for TemporalJoinExecutor { fn info(&self) -> &ExecutorInfo { diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index 74852fbf3c663..fa8c53482ad19 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -21,7 +21,7 @@ use tokio::sync::mpsc; use super::error::StreamExecutorError; use super::{ - Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, MessageStream, PkIndices, + Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, MessageStream, PkIndices, StreamChunk, StreamExecutorResult, Watermark, }; @@ -40,7 +40,7 @@ pub mod prelude { pub use crate::common::table::state_table::StateTable; pub use crate::executor::test_utils::expr::build_from_pretty; pub use crate::executor::test_utils::{MessageSender, MockSource, StreamExecutorTestExt}; - pub use crate::executor::{ActorContext, BoxedMessageStream, Executor, PkIndices}; + pub use crate::executor::{ActorContext, BoxedMessageStream, Execute, PkIndices}; } pub struct MockSource { @@ -179,7 +179,7 @@ impl MockSource { } } -impl Executor for MockSource { +impl Execute for MockSource { fn info(&self) -> &ExecutorInfo { &self.info } @@ -298,14 +298,14 @@ pub mod agg_executor { }; use crate::executor::aggregation::AggStateStorage; use crate::executor::{ - ActorContext, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, HashAggExecutor, + ActorContext, ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, HashAggExecutor, PkIndices, SimpleAggExecutor, }; - /// Generate agg executor's schema from `input`, `agg_calls` and `group_key_indices`. + /// Generate aggExecuter's schema from `input`, `agg_calls` and `group_key_indices`. /// For [`crate::executor::HashAggExecutor`], the group key indices should be provided. pub fn generate_agg_schema( - input: &dyn Executor, + input: &dyn Execute, agg_calls: &[AggCall], group_key_indices: Option<&[usize]>, ) -> Schema { @@ -334,7 +334,7 @@ pub mod agg_executor { agg_call: &AggCall, group_key_indices: &[usize], pk_indices: &[usize], - input_ref: &dyn Executor, + input_ref: &dyn Execute, is_append_only: bool, ) -> AggStateStorage { match agg_call.kind { @@ -405,7 +405,7 @@ pub mod agg_executor { table_id: TableId, agg_calls: &[AggCall], group_key_indices: &[usize], - input_ref: &dyn Executor, + input_ref: &dyn Execute, ) -> StateTable { let input_fields = input_ref.schema().fields(); @@ -444,7 +444,7 @@ pub mod agg_executor { #[allow(clippy::too_many_arguments)] pub async fn new_boxed_hash_agg_executor( store: S, - input: Box, + input: Box, is_append_only: bool, agg_calls: Vec, row_count_index: usize, @@ -453,7 +453,7 @@ pub mod agg_executor { extreme_cache_size: usize, emit_on_window_close: bool, executor_id: u64, - ) -> Box { + ) -> Box { let mut storages = Vec::with_capacity(agg_calls.iter().len()); for (idx, agg_call) in agg_calls.iter().enumerate() { storages.push( @@ -523,7 +523,7 @@ pub mod agg_executor { row_count_index: usize, pk_indices: PkIndices, executor_id: u64, - ) -> Box { + ) -> Box { let storages = future::join_all(agg_calls.iter().enumerate().map(|(idx, agg_call)| { create_agg_state_storage( store.clone(), diff --git a/src/stream/src/executor/top_n/group_top_n.rs b/src/stream/src/executor/top_n/group_top_n.rs index ed0f64d098a7a..eb9889abf25fb 100644 --- a/src/stream/src/executor/top_n/group_top_n.rs +++ b/src/stream/src/executor/top_n/group_top_n.rs @@ -32,7 +32,7 @@ use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; use crate::task::AtomicU64Ref; pub type GroupTopNExecutor = @@ -41,7 +41,7 @@ pub type GroupTopNExecutor = impl GroupTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Box, ctx: ActorContextRef, info: ExecutorInfo, storage_key: Vec, @@ -395,7 +395,7 @@ mod tests { }; let top_n_executor = Box::new( GroupTopNExecutor::::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -497,7 +497,7 @@ mod tests { }; let top_n_executor = Box::new( GroupTopNExecutor::::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -592,7 +592,7 @@ mod tests { }; let top_n_executor = Box::new( GroupTopNExecutor::::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), diff --git a/src/stream/src/executor/top_n/group_top_n_appendonly.rs b/src/stream/src/executor/top_n/group_top_n_appendonly.rs index 44522ddfb31ff..edbe54a7ff7ac 100644 --- a/src/stream/src/executor/top_n/group_top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/group_top_n_appendonly.rs @@ -31,7 +31,7 @@ use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; use crate::task::AtomicU64Ref; /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need @@ -45,7 +45,7 @@ impl { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Box, ctx: ActorContextRef, info: ExecutorInfo, storage_key: Vec, diff --git a/src/stream/src/executor/top_n/top_n_appendonly.rs b/src/stream/src/executor/top_n/top_n_appendonly.rs index 8da016fd8c7ac..7fce9c7adfd1d 100644 --- a/src/stream/src/executor/top_n/top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/top_n_appendonly.rs @@ -24,7 +24,7 @@ use super::{ManagedTopNState, TopNCache, NO_GROUP_KEY}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need /// to keep all the rows seen. As long as a record @@ -38,7 +38,7 @@ pub type AppendOnlyTopNExecutor = impl AppendOnlyTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Box, ctx: ActorContextRef, info: ExecutorInfo, storage_key: Vec, @@ -170,7 +170,7 @@ mod tests { use super::AppendOnlyTopNExecutor; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Barrier, Executor, ExecutorInfo, Message, PkIndices}; + use crate::executor::{ActorContext, Barrier, Execute, ExecutorInfo, Message, PkIndices}; fn create_stream_chunks() -> Vec { let chunk1 = StreamChunk::from_pretty( @@ -258,7 +258,7 @@ mod tests { }; let top_n_executor = Box::new( AppendOnlyTopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key, @@ -345,7 +345,7 @@ mod tests { }; let top_n_executor = Box::new( AppendOnlyTopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs index 8536d9d3273c3..7ab0f954199be 100644 --- a/src/stream/src/executor/top_n/top_n_plain.rs +++ b/src/stream/src/executor/top_n/top_n_plain.rs @@ -23,7 +23,7 @@ use super::{ManagedTopNState, TopNCache, TopNCacheTrait}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; /// `TopNExecutor` works with input with modification, it keeps all the data /// records/rows that have been seen, and returns topN records overall. @@ -33,7 +33,7 @@ pub type TopNExecutor = impl TopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Box, ctx: ActorContextRef, info: ExecutorInfo, storage_key: Vec, @@ -61,7 +61,7 @@ impl TopNExecutor { #[allow(clippy::too_many_arguments)] #[cfg(test)] pub fn new_with_ties_for_test( - input: Box, + input: Box, ctx: ActorContextRef, info: ExecutorInfo, storage_key: Vec, @@ -302,7 +302,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -403,7 +403,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -516,7 +516,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::<_, true>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -628,7 +628,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -860,7 +860,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -944,7 +944,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -1004,7 +1004,7 @@ mod tests { }; let top_n_executor_after_recovery = Box::new( TopNExecutor::<_, false>::new( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -1138,7 +1138,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::new_with_ties_for_test( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -1294,7 +1294,7 @@ mod tests { }; let top_n_executor = Box::new( TopNExecutor::new_with_ties_for_test( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), @@ -1357,7 +1357,7 @@ mod tests { }; let top_n_executor_after_recovery = Box::new( TopNExecutor::new_with_ties_for_test( - source as Box, + source as Box, ActorContext::for_test(0), info, storage_key(), diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index dca30b0f6104d..aad7e23b53169 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -30,7 +30,7 @@ use risingwave_common::util::sort_util::ColumnOrder; use super::CacheKey; use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, + expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, Watermark, }; @@ -77,7 +77,7 @@ pub struct TopNExecutorWrapper { pub(super) inner: E, } -impl Executor for TopNExecutorWrapper +impl Execute for TopNExecutorWrapper where E: TopNExecutorBase, { diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index 76fd0d698038c..a2c51c92c6265 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -21,7 +21,6 @@ use futures::StreamExt; use futures_async_stream::try_stream; use pin_project::pin_project; - use super::watermark::BufferedWatermarks; use super::*; use crate::executor::{BoxedMessageStream, ExecutorInfo}; @@ -47,7 +46,7 @@ impl UnionExecutor { } } -impl Executor for UnionExecutor { +impl Execute for UnionExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs index eae687da4dabb..9ed9e3d8ae602 100644 --- a/src/stream/src/executor/utils.rs +++ b/src/stream/src/executor/utils.rs @@ -13,11 +13,10 @@ // limitations under the License. use futures::StreamExt; - use risingwave_common::metrics::LabelGuardedIntCounter; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{BoxedMessageStream, Executor, ExecutorInfo}; +use crate::executor::{BoxedMessageStream, Execute, ExecutorInfo}; use crate::task::{ActorId, FragmentId}; #[derive(Default)] @@ -31,7 +30,7 @@ impl DummyExecutor { } } -impl Executor for DummyExecutor { +impl Execute for DummyExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index 0c893c423431c..c5e2097f8102a 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -18,14 +18,13 @@ use await_tree::InstrumentAwait; use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::{DataChunk, Op, StreamChunk}; - use risingwave_common::ensure; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; use tokio::sync::mpsc::UnboundedReceiver; use super::{ - ActorContextRef, Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, + ActorContextRef, Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, StreamExecutorError, }; use crate::task::CreateMviewProgress; @@ -135,7 +134,7 @@ impl ValuesExecutor { } } -impl Executor for ValuesExecutor { +impl Execute for ValuesExecutor { fn info(&self) -> &ExecutorInfo { &self.info } @@ -159,7 +158,7 @@ mod tests { use super::ValuesExecutor; use crate::executor::test_utils::StreamExecutorTestExt; - use crate::executor::{ActorContext, AddMutation, Barrier, Executor, ExecutorInfo, Mutation}; + use crate::executor::{ActorContext, AddMutation, Barrier, Execute, ExecutorInfo, Mutation}; use crate::task::{CreateMviewProgress, LocalBarrierManager}; #[tokio::test] diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 37b2f1e41b564..7d9eca0c7ed06 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -36,9 +36,7 @@ use risingwave_storage::StateStore; use super::error::StreamExecutorError; use super::filter::FilterExecutor; -use super::{ - ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, StreamExecutorResult, -}; +use super::{ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message, StreamExecutorResult}; use crate::common::table::state_table::StateTable; use crate::executor::{expect_first_barrier, Watermark}; use crate::task::ActorEvalErrorReport; @@ -81,7 +79,7 @@ impl WatermarkFilterExecutor { } } -impl Executor for WatermarkFilterExecutor { +impl Execute for WatermarkFilterExecutor { fn info(&self) -> &ExecutorInfo { &self.info } diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index 3acbafe509dd2..e671f204f5a61 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -16,7 +16,6 @@ use std::sync::Arc; use futures::StreamExt; - use super::*; mod epoch_check; @@ -88,7 +87,7 @@ impl WrapperExecutor { } } -impl Executor for WrapperExecutor { +impl Execute for WrapperExecutor { fn info(&self) -> &ExecutorInfo { self.input.info() } diff --git a/src/stream/src/executor/wrapper/epoch_check.rs b/src/stream/src/executor/wrapper/epoch_check.rs index 1477b3081f3f9..ab31420011397 100644 --- a/src/stream/src/executor/wrapper/epoch_check.rs +++ b/src/stream/src/executor/wrapper/epoch_check.rs @@ -80,7 +80,7 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; + use crate::executor::Execute; #[tokio::test] async fn test_epoch_ok() { diff --git a/src/stream/src/executor/wrapper/schema_check.rs b/src/stream/src/executor/wrapper/schema_check.rs index 7d08c2dcc6b90..17ae1baca11df 100644 --- a/src/stream/src/executor/wrapper/schema_check.rs +++ b/src/stream/src/executor/wrapper/schema_check.rs @@ -62,7 +62,7 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; + use crate::executor::Execute; #[tokio::test] async fn test_schema_ok() { diff --git a/src/stream/src/executor/wrapper/update_check.rs b/src/stream/src/executor/wrapper/update_check.rs index d880249de8dfb..ae98e265285ff 100644 --- a/src/stream/src/executor/wrapper/update_check.rs +++ b/src/stream/src/executor/wrapper/update_check.rs @@ -62,7 +62,7 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; + use crate::executor::Execute; #[should_panic] #[tokio::test] diff --git a/src/stream/src/from_proto/eowc_over_window.rs b/src/stream/src/from_proto/eowc_over_window.rs index b2136f51c3970..26879473dfbfc 100644 --- a/src/stream/src/from_proto/eowc_over_window.rs +++ b/src/stream/src/from_proto/eowc_over_window.rs @@ -21,9 +21,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{ - BoxedExecutor, EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Executor, -}; +use crate::executor::{BoxedExecutor, EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Execute}; use crate::task::ExecutorParams; pub struct EowcOverWindowExecutorBuilder; diff --git a/src/stream/src/from_proto/hash_join.rs b/src/stream/src/from_proto/hash_join.rs index d04db948853ba..0199e804c1f28 100644 --- a/src/stream/src/from_proto/hash_join.rs +++ b/src/stream/src/from_proto/hash_join.rs @@ -164,8 +164,8 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { struct HashJoinExecutorDispatcherArgs { ctx: ActorContextRef, info: ExecutorInfo, - source_l: Box, - source_r: Box, + source_l: BoxedExecutor, + source_r: BoxedExecutor, params_l: JoinParams, params_r: JoinParams, null_safe: Vec, diff --git a/src/stream/src/from_proto/mod.rs b/src/stream/src/from_proto/mod.rs index 46ec1a6cc8e59..9d87603ce03e7 100644 --- a/src/stream/src/from_proto/mod.rs +++ b/src/stream/src/from_proto/mod.rs @@ -93,7 +93,7 @@ use self::top_n::*; use self::union::*; use self::watermark_filter::WatermarkFilterBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Executor, ExecutorInfo}; +use crate::executor::{BoxedExecutor, Execute, ExecutorInfo}; use crate::from_proto::subscription::SubscriptionExecutorBuilder; use crate::from_proto::values::ValuesExecutorBuilder; use crate::task::ExecutorParams; diff --git a/src/stream/src/from_proto/no_op.rs b/src/stream/src/from_proto/no_op.rs index 606b584d9109c..22dae0020abf5 100644 --- a/src/stream/src/from_proto/no_op.rs +++ b/src/stream/src/from_proto/no_op.rs @@ -17,7 +17,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Executor, NoOpExecutor}; +use crate::executor::{BoxedExecutor, Execute, NoOpExecutor}; use crate::task::ExecutorParams; pub struct NoOpExecutorBuilder; diff --git a/src/stream/src/from_proto/over_window.rs b/src/stream/src/from_proto/over_window.rs index d2fc2893b6135..ddc5db3cc2582 100644 --- a/src/stream/src/from_proto/over_window.rs +++ b/src/stream/src/from_proto/over_window.rs @@ -23,7 +23,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Executor, OverWindowExecutor, OverWindowExecutorArgs}; +use crate::executor::{BoxedExecutor, Execute, OverWindowExecutor, OverWindowExecutorArgs}; use crate::task::ExecutorParams; pub struct OverWindowExecutorBuilder; diff --git a/src/stream/src/from_proto/source/fs_fetch.rs b/src/stream/src/from_proto/source/fs_fetch.rs index 97cf26129d8d5..fa2f4475b3759 100644 --- a/src/stream/src/from_proto/source/fs_fetch.rs +++ b/src/stream/src/from_proto/source/fs_fetch.rs @@ -25,7 +25,7 @@ use risingwave_storage::StateStore; use crate::error::StreamResult; use crate::executor::{ - BoxedExecutor, Executor, FlowControlExecutor, FsFetchExecutor, SourceStateTableHandler, + BoxedExecutor, Execute, FlowControlExecutor, FsFetchExecutor, SourceStateTableHandler, StreamSourceCore, }; use crate::from_proto::ExecutorBuilder; From f4bd48d1378c765b8d295635f3c0a005e245596d Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Wed, 21 Feb 2024 00:31:45 +0800 Subject: [PATCH 08/13] introduce `struct Executor` Signed-off-by: Richard Chien --- src/stream/src/executor/agg_common.rs | 4 +- .../executor/backfill/arrangement_backfill.rs | 16 ++--- .../src/executor/backfill/cdc/cdc_backfill.rs | 15 ++-- .../executor/backfill/no_shuffle_backfill.rs | 16 ++--- src/stream/src/executor/barrier_recv.rs | 22 +----- src/stream/src/executor/batch_query.rs | 15 ++-- src/stream/src/executor/chain.rs | 18 ++--- .../src/executor/dedup/append_only_dedup.rs | 28 ++++---- src/stream/src/executor/dispatch.rs | 8 +-- src/stream/src/executor/dml.rs | 16 +---- src/stream/src/executor/dynamic_filter.rs | 39 ++++++----- src/stream/src/executor/expand.rs | 18 ++--- src/stream/src/executor/filter.rs | 22 +++--- src/stream/src/executor/flow_control.rs | 28 ++------ src/stream/src/executor/hash_agg.rs | 10 ++- src/stream/src/executor/hash_join.rs | 14 ++-- src/stream/src/executor/hop_window.rs | 13 +--- src/stream/src/executor/integration_tests.rs | 4 +- src/stream/src/executor/lookup.rs | 10 +-- src/stream/src/executor/lookup/impl_.rs | 6 +- src/stream/src/executor/lookup/sides.rs | 12 +--- src/stream/src/executor/lookup_union.rs | 30 ++++---- src/stream/src/executor/merge.rs | 14 ---- src/stream/src/executor/mod.rs | 70 ++++++++++++++----- src/stream/src/executor/mview/materialize.rs | 36 ++++------ src/stream/src/executor/no_op.rs | 17 ++--- src/stream/src/executor/now.rs | 23 ++---- src/stream/src/executor/over_window/eowc.rs | 23 +++--- .../src/executor/over_window/general.rs | 22 +++--- src/stream/src/executor/project.rs | 26 ++++--- src/stream/src/executor/project_set.rs | 21 ++---- src/stream/src/executor/rearranged_chain.rs | 22 ++---- src/stream/src/executor/receiver.rs | 21 +----- src/stream/src/executor/row_id_gen.rs | 13 +--- src/stream/src/executor/simple_agg.rs | 6 +- src/stream/src/executor/sink.rs | 10 +-- src/stream/src/executor/sort.rs | 21 +++--- .../src/executor/source/fetch_executor.rs | 21 +++--- .../src/executor/source/fs_source_executor.rs | 10 +-- .../src/executor/source/list_executor.rs | 10 +-- .../src/executor/source/source_executor.rs | 13 +--- .../src/executor/stateless_simple_agg.rs | 18 ++--- src/stream/src/executor/subscription.rs | 15 ++-- src/stream/src/executor/subtask.rs | 8 +-- src/stream/src/executor/temporal_join.rs | 16 ++--- src/stream/src/executor/test_utils.rs | 52 +++++++------- src/stream/src/executor/top_n/group_top_n.rs | 26 +++---- .../executor/top_n/group_top_n_appendonly.rs | 26 +++---- .../src/executor/top_n/top_n_appendonly.rs | 27 ++++--- src/stream/src/executor/top_n/top_n_plain.rs | 25 +++---- src/stream/src/executor/top_n/utils.rs | 12 +--- src/stream/src/executor/union.rs | 25 ++++--- src/stream/src/executor/utils.rs | 14 +--- src/stream/src/executor/values.rs | 26 +++---- src/stream/src/executor/watermark_filter.rs | 31 ++++---- src/stream/src/executor/wrapper.rs | 10 +-- .../src/from_proto/append_only_dedup.rs | 13 ++-- src/stream/src/from_proto/barrier_recv.rs | 5 +- src/stream/src/from_proto/batch_query.rs | 14 ++-- src/stream/src/from_proto/cdc_filter.rs | 5 +- src/stream/src/from_proto/dml.rs | 10 +-- src/stream/src/from_proto/dynamic_filter.rs | 22 +++--- src/stream/src/from_proto/eowc_over_window.rs | 12 ++-- src/stream/src/from_proto/expand.rs | 4 +- src/stream/src/from_proto/filter.rs | 5 +- src/stream/src/from_proto/group_top_n.rs | 17 ++--- src/stream/src/from_proto/hash_agg.rs | 11 +-- src/stream/src/from_proto/hash_join.rs | 58 +++++++-------- src/stream/src/from_proto/hop_window.rs | 23 ++---- src/stream/src/from_proto/lookup.rs | 9 +-- src/stream/src/from_proto/lookup_union.rs | 5 +- src/stream/src/from_proto/merge.rs | 17 +++-- src/stream/src/from_proto/mod.rs | 10 +-- src/stream/src/from_proto/mview.rs | 16 ++--- src/stream/src/from_proto/no_op.rs | 6 +- src/stream/src/from_proto/now.rs | 11 +-- src/stream/src/from_proto/over_window.rs | 12 ++-- src/stream/src/from_proto/project.rs | 9 ++- src/stream/src/from_proto/project_set.rs | 9 ++- src/stream/src/from_proto/row_id_gen.rs | 9 ++- src/stream/src/from_proto/simple_agg.rs | 11 +-- src/stream/src/from_proto/sink.rs | 54 +++++++------- src/stream/src/from_proto/sort.rs | 9 +-- src/stream/src/from_proto/source/fs_fetch.rs | 15 ++-- .../src/from_proto/source/trad_source.rs | 21 +++--- .../src/from_proto/stateless_simple_agg.rs | 13 ++-- src/stream/src/from_proto/stream_cdc_scan.rs | 20 +++--- src/stream/src/from_proto/stream_scan.rs | 21 +++--- src/stream/src/from_proto/subscription.rs | 22 +++--- src/stream/src/from_proto/temporal_join.rs | 12 ++-- src/stream/src/from_proto/top_n.rs | 9 +-- src/stream/src/from_proto/union.rs | 4 +- src/stream/src/from_proto/values.rs | 11 +-- src/stream/src/from_proto/watermark_filter.rs | 10 +-- src/stream/src/task/stream_manager.rs | 46 +++++------- 95 files changed, 703 insertions(+), 981 deletions(-) diff --git a/src/stream/src/executor/agg_common.rs b/src/stream/src/executor/agg_common.rs index fa7a3f9f93a9b..91c414877a76b 100644 --- a/src/stream/src/executor/agg_common.rs +++ b/src/stream/src/executor/agg_common.rs @@ -19,7 +19,7 @@ use risingwave_pb::stream_plan::PbAggNodeVersion; use risingwave_storage::StateStore; use super::aggregation::AggStateStorage; -use super::{Execute, ExecutorInfo}; +use super::{Executor, ExecutorInfo}; use crate::common::table::state_table::StateTable; use crate::executor::ActorContextRef; use crate::task::AtomicU64Ref; @@ -29,7 +29,7 @@ pub struct AggExecutorArgs { pub version: PbAggNodeVersion, // basic - pub input: Box, + pub input: Executor, pub actor_ctx: ActorContextRef, pub info: ExecutorInfo, diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 223d15865f966..5ce65448c0160 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -39,8 +39,8 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, - HashMap, Message, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, HashMap, + Message, StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; @@ -56,7 +56,7 @@ pub struct ArrangementBackfillExecutor { upstream_table: ReplicatedStateTable, /// Upstream with the same schema with the upstream table. - upstream: BoxedExecutor, + upstream: Executor, /// Internal state table for persisting state of backfill state. state_table: StateTable, @@ -68,8 +68,6 @@ pub struct ArrangementBackfillExecutor { actor_id: ActorId, - info: ExecutorInfo, - metrics: Arc, chunk_size: usize, @@ -85,9 +83,8 @@ where #[allow(clippy::too_many_arguments)] #[allow(dead_code)] pub fn new( - info: ExecutorInfo, upstream_table: ReplicatedStateTable, - upstream: BoxedExecutor, + upstream: Executor, state_table: StateTable, output_indices: Vec, progress: CreateMviewProgress, @@ -96,7 +93,6 @@ where rate_limit: Option, ) -> Self { Self { - info, upstream_table, upstream, state_table, @@ -714,10 +710,6 @@ where S: StateStore, SD: ValueRowSerde, { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index 988a69fb2ead2..c7a5180d42ecc 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -45,8 +45,8 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, - ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, StreamExecutorResult, }; use crate::task::CreateMviewProgress; @@ -55,13 +55,12 @@ const METADATA_STATE_LEN: usize = 4; pub struct CdcBackfillExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// The external table to be backfilled external_table: ExternalStorageTable, /// Upstream changelog stream which may contain metadata columns, e.g. `_rw_offset` - upstream: BoxedExecutor, + upstream: Executor, /// The column indices need to be forwarded to the downstream from the upstream and table scan. /// User may select a subset of columns from the upstream table. @@ -83,9 +82,8 @@ impl CdcBackfillExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, external_table: ExternalStorageTable, - upstream: BoxedExecutor, + upstream: Executor, output_indices: Vec, progress: Option, metrics: Arc, @@ -95,7 +93,6 @@ impl CdcBackfillExecutor { ) -> Self { Self { actor_ctx, - info, external_table, upstream, output_indices, @@ -610,10 +607,6 @@ fn get_rw_columns(schema: &Schema) -> Vec { } impl Execute for CdcBackfillExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index ad8e335b159a7..6b229baf7ad2a 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -38,8 +38,8 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, - Message, Mutation, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, + Mutation, StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; @@ -75,12 +75,10 @@ pub struct BackfillState { /// in the same worker, so that we can read uncommitted data from the upstream table without /// waiting. pub struct BackfillExecutor { - info: ExecutorInfo, - /// Upstream table upstream_table: StorageTable, /// Upstream with the same schema with the upstream table. - upstream: BoxedExecutor, + upstream: Executor, /// Internal state table for persisting state of backfill state. state_table: Option>, @@ -109,9 +107,8 @@ where { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, upstream_table: StorageTable, - upstream: BoxedExecutor, + upstream: Executor, state_table: Option>, output_indices: Vec, progress: CreateMviewProgress, @@ -121,7 +118,6 @@ where ) -> Self { let actor_id = progress.actor_id(); Self { - info, upstream_table, upstream, state_table, @@ -745,10 +741,6 @@ impl Execute for BackfillExecutor where S: StateStore, { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/barrier_recv.rs b/src/stream/src/executor/barrier_recv.rs index 4fce4cc34d545..a230c5fa952c7 100644 --- a/src/stream/src/executor/barrier_recv.rs +++ b/src/stream/src/executor/barrier_recv.rs @@ -26,43 +26,25 @@ use super::{ /// of the streaming graph. pub struct BarrierRecvExecutor { _ctx: ActorContextRef, - info: ExecutorInfo, /// The barrier receiver registered in the local barrier manager. barrier_receiver: UnboundedReceiver, } impl BarrierRecvExecutor { - pub fn new( - ctx: ActorContextRef, - info: ExecutorInfo, - barrier_receiver: UnboundedReceiver, - ) -> Self { + pub fn new(ctx: ActorContextRef, barrier_receiver: UnboundedReceiver) -> Self { Self { _ctx: ctx, - info, barrier_receiver, } } pub fn for_test(barrier_receiver: UnboundedReceiver) -> Self { - Self::new( - ActorContext::for_test(0), - ExecutorInfo { - schema: Schema::empty().clone(), - pk_indices: PkIndices::new(), - identity: "BarrierRecvExecutor".to_string(), - }, - barrier_receiver, - ) + Self::new(ActorContext::for_test(0), barrier_receiver) } } impl Execute for BarrierRecvExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { UnboundedReceiverStream::new(self.barrier_receiver) .map(|barrier| Ok(Message::Barrier(barrier))) diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs index 5fc2607179d52..3e2ac96f796cf 100644 --- a/src/stream/src/executor/batch_query.rs +++ b/src/stream/src/executor/batch_query.rs @@ -16,6 +16,7 @@ use await_tree::InstrumentAwait; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{Op, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_hummock_sdk::HummockReadEpoch; use risingwave_storage::store::PrefetchOptions; use risingwave_storage::table::batch_table::storage_table::StorageTable; @@ -23,7 +24,7 @@ use risingwave_storage::table::collect_data_chunk; use risingwave_storage::StateStore; use super::error::StreamExecutorError; -use super::{Execute, ExecutorInfo, Message}; +use super::{Execute, Message}; use crate::executor::BoxedMessageStream; pub struct BatchQueryExecutor { @@ -33,18 +34,18 @@ pub struct BatchQueryExecutor { /// The number of tuples in one [`StreamChunk`] batch_size: usize, - info: ExecutorInfo, + schema: Schema, } impl BatchQueryExecutor where S: StateStore, { - pub fn new(table: StorageTable, batch_size: usize, info: ExecutorInfo) -> Self { + pub fn new(table: StorageTable, batch_size: usize, schema: Schema) -> Self { Self { table, batch_size, - info, + schema, } } @@ -61,7 +62,7 @@ where pin_mut!(iter); while let Some(data_chunk) = - collect_data_chunk(&mut iter, self.schema(), Some(self.batch_size)) + collect_data_chunk(&mut iter, &self.schema, Some(self.batch_size)) .instrument_await("batch_query_executor_collect_chunk") .await? { @@ -76,10 +77,6 @@ impl Execute for BatchQueryExecutor where S: StateStore, { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { unreachable!("should call `execute_with_epoch`") } diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index ca40405302fb8..d42da1995115c 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -16,7 +16,7 @@ use futures::StreamExt; use futures_async_stream::try_stream; use super::error::StreamExecutorError; -use super::{expect_first_barrier, BoxedExecutor, Execute, ExecutorInfo, Message}; +use super::{expect_first_barrier, Execute, Executor, Message}; use crate::task::{ActorId, CreateMviewProgress}; /// [`ChainExecutor`] is an executor that enables synchronization between the existing stream and @@ -24,11 +24,9 @@ use crate::task::{ActorId, CreateMviewProgress}; /// feature. It pipes new data of existing MVs to newly created MV only all of the old data in the /// existing MVs are dispatched. pub struct ChainExecutor { - info: ExecutorInfo, + snapshot: Executor, - snapshot: BoxedExecutor, - - upstream: BoxedExecutor, + upstream: Executor, progress: CreateMviewProgress, @@ -40,14 +38,12 @@ pub struct ChainExecutor { impl ChainExecutor { pub fn new( - info: ExecutorInfo, - snapshot: BoxedExecutor, - upstream: BoxedExecutor, + snapshot: Executor, + upstream: Executor, progress: CreateMviewProgress, upstream_only: bool, ) -> Self { Self { - info, snapshot, upstream, actor_id: progress.actor_id(), @@ -104,10 +100,6 @@ impl ChainExecutor { } impl Execute for ChainExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index 73650f7cf0c2f..4154ab1bc45c4 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -28,39 +28,39 @@ use crate::common::table::state_table::StateTable; use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, - ExecutorInfo, Message, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorResult, }; use crate::task::AtomicU64Ref; /// [`AppendOnlyDedupExecutor`] drops any message that has duplicate pk columns with previous /// messages. It only accepts append-only input, and its output will be append-only as well. pub struct AppendOnlyDedupExecutor { - input: Option, + ctx: ActorContextRef, + + input: Option, + dedup_cols: Vec, state_table: StateTable, cache: DedupCache, - - info: ExecutorInfo, - ctx: ActorContextRef, } impl AppendOnlyDedupExecutor { pub fn new( - input: BoxedExecutor, - state_table: StateTable, - info: ExecutorInfo, ctx: ActorContextRef, + input: Executor, + dedup_cols: Vec, + state_table: StateTable, watermark_epoch: AtomicU64Ref, metrics: Arc, ) -> Self { let metrics_info = MetricsInfo::new(metrics, state_table.table_id(), ctx.id, "AppendOnly Dedup"); Self { + ctx, input: Some(input), + dedup_cols, state_table, cache: DedupCache::new(watermark_epoch, metrics_info), - info, - ctx, } } @@ -89,7 +89,7 @@ impl AppendOnlyDedupExecutor { .data_chunk() .rows_with_holes() .map(|row_ref| { - row_ref.map(|row| row.project(self.pk_indices()).to_owned_row()) + row_ref.map(|row| row.project(&self.dedup_cols).to_owned_row()) }) .collect_vec(); @@ -185,10 +185,6 @@ impl AppendOnlyDedupExecutor { } impl Execute for AppendOnlyDedupExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.executor_inner().boxed() } diff --git a/src/stream/src/executor/dispatch.rs b/src/stream/src/executor/dispatch.rs index a21012c5f1664..8a6476afff910 100644 --- a/src/stream/src/executor/dispatch.rs +++ b/src/stream/src/executor/dispatch.rs @@ -35,17 +35,17 @@ use tokio::time::Instant; use tracing::{event, Instrument}; use super::exchange::output::{new_output, BoxedOutput}; -use super::{AddMutation, UpdateMutation, Watermark}; +use super::{AddMutation, Executor, UpdateMutation, Watermark}; use crate::error::StreamResult; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{Barrier, BoxedExecutor, Message, Mutation, StreamConsumer}; +use crate::executor::{Barrier, Message, Mutation, StreamConsumer}; use crate::task::{ActorId, DispatcherId, SharedContext}; /// [`DispatchExecutor`] consumes messages and send them into downstream actors. Usually, /// data chunks will be dispatched with some specified policy, while control message /// such as barriers will be distributed to all receivers. pub struct DispatchExecutor { - input: BoxedExecutor, + input: Executor, inner: DispatchExecutorInner, } @@ -341,7 +341,7 @@ impl DispatchExecutorInner { impl DispatchExecutor { pub fn new( - input: BoxedExecutor, + input: Executor, dispatchers: Vec, actor_id: u32, fragment_id: u32, diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index 96db02a08b09d..9ac93f0368872 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -26,8 +26,7 @@ use risingwave_dml::dml_manager::DmlManagerRef; use super::error::StreamExecutorError; use super::{ - expect_first_barrier, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, - Mutation, + expect_first_barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, Mutation, }; use crate::common::StreamChunkBuilder; use crate::executor::stream_reader::StreamReaderWithPause; @@ -35,9 +34,7 @@ use crate::executor::stream_reader::StreamReaderWithPause; /// [`DmlExecutor`] accepts both stream data and batch data for data manipulation on a specific /// table. The two streams will be merged into one and then sent to downstream. pub struct DmlExecutor { - info: ExecutorInfo, - - upstream: BoxedExecutor, + upstream: Executor, /// Stores the information of batch data channels. dml_manager: DmlManagerRef, @@ -71,10 +68,8 @@ struct TxnBuffer { } impl DmlExecutor { - #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, - upstream: BoxedExecutor, + upstream: Executor, dml_manager: DmlManagerRef, table_id: TableId, table_version_id: TableVersionId, @@ -82,7 +77,6 @@ impl DmlExecutor { chunk_size: usize, ) -> Self { Self { - info, upstream, dml_manager, table_id, @@ -278,10 +272,6 @@ impl DmlExecutor { } impl Execute for DmlExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index fd660db379151..d34fd612c966b 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -20,6 +20,7 @@ use futures_async_stream::try_stream; use risingwave_common::array::{Array, ArrayImpl, Op, StreamChunk}; use risingwave_common::bail; use risingwave_common::buffer::{Bitmap, BitmapBuilder}; +use risingwave_common::catalog::Schema; use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{self, once, OwnedRow, OwnedRow as RowData, Row}; use risingwave_common::types::{DataType, Datum, DefaultOrd, ScalarImpl, ToDatumRef, ToOwnedDatum}; @@ -37,7 +38,7 @@ use risingwave_storage::StateStore; use super::barrier_align::*; use super::error::StreamExecutorError; use super::monitor::StreamingMetrics; -use super::{ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message}; +use super::{ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message}; use crate::common::table::state_table::{StateTable, WatermarkCacheParameterizedStateTable}; use crate::common::StreamChunkBuilder; use crate::executor::expect_first_barrier_from_aligned_stream; @@ -45,10 +46,12 @@ use crate::task::ActorEvalErrorReport; pub struct DynamicFilterExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - source_l: Option, - source_r: Option, + eval_error_report: ActorEvalErrorReport, + + schema: Schema, + source_l: Option, + source_r: Option, key_l: usize, comparator: ExprNodeType, left_table: WatermarkCacheParameterizedStateTable, @@ -67,9 +70,9 @@ impl DynamicFilterExecutor, @@ -79,9 +82,14 @@ impl DynamicFilterExecutor Self { + let eval_error_report = ActorEvalErrorReport { + actor_context: ctx.clone(), + identity: Arc::from(info.identity.as_str()), + }; Self { ctx, - info, + eval_error_report, + schema: info.schema.clone(), source_l: Some(source_l), source_r: Some(source_r), key_l, @@ -259,7 +267,7 @@ impl DynamicFilterExecutor DynamicFilterExecutor DynamicFilterExecutor DynamicFilterExecutor Execute for DynamicFilterExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() + self.execute_inner().boxed() } } diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index 3af392ca3164b..32546dc45d6fe 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -16,25 +16,19 @@ use std::fmt::Debug; use futures::StreamExt; use futures_async_stream::try_stream; -use risingwave_common::array::{Array, I64Array}; +use risingwave_common::array::{Array, I64Array, StreamChunk}; use super::error::StreamExecutorError; -use super::*; +use super::{BoxedMessageStream, Execute, Executor, ExecutorInfo, Message}; pub struct ExpandExecutor { - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, column_subsets: Vec>, } impl ExpandExecutor { - pub fn new( - info: ExecutorInfo, - input: Box, - column_subsets: Vec>, - ) -> Self { + pub fn new(input: Executor, column_subsets: Vec>) -> Self { Self { - info, input, column_subsets, } @@ -73,10 +67,6 @@ impl Debug for ExpandExecutor { } impl Execute for ExpandExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index a6bce873d7d96..6ec6091eaa3c7 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -15,12 +15,17 @@ use std::fmt::{Debug, Formatter}; use std::sync::Arc; +use futures::StreamExt; +use futures_async_stream::try_stream; use risingwave_common::array::{Array, ArrayImpl, Op, StreamChunk}; use risingwave_common::buffer::BitmapBuilder; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; -use super::*; +use super::{ + ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, + StreamExecutorError, StreamExecutorResult, +}; /// `FilterExecutor` filters data with the `expr`. The `expr` takes a chunk of data, /// and returns a boolean array on whether each item should be retained. And then, @@ -28,8 +33,7 @@ use super::*; /// to the result of the expression. pub struct FilterExecutor { _ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, /// Expression of the current filter, note that the filter must always have the same output for /// the same input. @@ -37,15 +41,9 @@ pub struct FilterExecutor { } impl FilterExecutor { - pub fn new( - ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, - expr: NonStrictExpression, - ) -> Self { + pub fn new(ctx: ActorContextRef, input: Executor, expr: NonStrictExpression) -> Self { Self { _ctx: ctx, - info, input, expr, } @@ -135,10 +133,6 @@ impl Debug for FilterExecutor { } impl Execute for FilterExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs index 1719611507c35..4e4212a1e246e 100644 --- a/src/stream/src/executor/flow_control.rs +++ b/src/stream/src/executor/flow_control.rs @@ -15,10 +15,14 @@ use std::fmt::{Debug, Formatter}; use std::num::NonZeroU32; +use futures::StreamExt; +use futures_async_stream::try_stream; use governor::clock::MonotonicClock; use governor::{Quota, RateLimiter}; -use super::*; +use super::{ + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, Mutation, StreamExecutorError, +}; /// Flow Control Executor is used to control the rate of the input executor. /// @@ -30,28 +34,14 @@ use super::*; /// /// It is used to throttle problematic MVs that are consuming too much resources. pub struct FlowControlExecutor { - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, actor_ctx: ActorContextRef, rate_limit: Option, } impl FlowControlExecutor { - pub fn new( - input: Box, - actor_ctx: ActorContextRef, - rate_limit: Option, - ) -> Self { - let identity = if rate_limit.is_some() { - format!("{} (flow controlled)", input.identity()) - } else { - input.identity().to_owned() - }; + pub fn new(input: Executor, actor_ctx: ActorContextRef, rate_limit: Option) -> Self { Self { - info: ExecutorInfo { - identity, - ..input.info().clone() - }, input, actor_ctx, rate_limit, @@ -131,10 +121,6 @@ impl Debug for FlowControlExecutor { } impl Execute for FlowControlExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index bea9a8f696932..ccaa4d98ce11d 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -38,7 +38,9 @@ use super::aggregation::{ OnlyOutputIfHasInput, }; use super::sort_buffer::SortBuffer; -use super::{expect_first_barrier, ActorContextRef, ExecutorInfo, StreamExecutorResult, Watermark}; +use super::{ + expect_first_barrier, ActorContextRef, Executor, ExecutorInfo, StreamExecutorResult, Watermark, +}; use crate::cache::{cache_may_stale, new_with_hasher, ManagedLruCache}; use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; @@ -71,7 +73,7 @@ type AggGroupCache = ManagedLruCache>, Precompu /// all modifications will be flushed to the storage backend. Meanwhile, the executor will go /// through `group_change_set`, and produce a stream chunk based on the state changes. pub struct HashAggExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } @@ -191,10 +193,6 @@ impl ExecutionStats { } impl Execute for HashAggExecutor { - fn info(&self) -> &ExecutorInfo { - &self.inner.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index 9fb665f135dc0..aa3477ca5cb11 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -41,7 +41,7 @@ use super::join::{JoinTypePrimitive, SideTypePrimitive, *}; use super::monitor::StreamingMetrics; use super::watermark::*; use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo, Message, Watermark, + ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, Watermark, }; use crate::common::table::state_table::StateTable; use crate::executor::expect_first_barrier_from_aligned_stream; @@ -141,9 +141,9 @@ pub struct HashJoinExecutor, + input_l: Option, /// Right input executor - input_r: Option, + input_r: Option, /// The data types of the formed new columns actual_output_data_types: Vec, /// The parameters of the left join executor @@ -190,10 +190,6 @@ impl std::fmt::Debug } impl Execute for HashJoinExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } @@ -217,8 +213,8 @@ impl HashJoinExecutor, diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs index a192156205f85..1e928c929ae67 100644 --- a/src/stream/src/executor/hop_window.rs +++ b/src/stream/src/executor/hop_window.rs @@ -23,13 +23,12 @@ use risingwave_expr::expr::NonStrictExpression; use risingwave_expr::ExprError; use super::error::StreamExecutorError; -use super::{ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message}; +use super::{ActorContextRef, Execute, Executor, Message}; use crate::common::StreamChunkBuilder; pub struct HopWindowExecutor { _ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, pub time_col_idx: usize, pub window_slide: Interval, pub window_size: Interval, @@ -43,8 +42,7 @@ impl HopWindowExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, time_col_idx: usize, window_slide: Interval, window_size: Interval, @@ -55,7 +53,6 @@ impl HopWindowExecutor { ) -> Self { HopWindowExecutor { _ctx: ctx, - info, input, time_col_idx, window_slide, @@ -69,10 +66,6 @@ impl HopWindowExecutor { } impl Execute for HopWindowExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs index e3652cce0398f..258709955776f 100644 --- a/src/stream/src/executor/integration_tests.rs +++ b/src/stream/src/executor/integration_tests.rs @@ -232,7 +232,7 @@ async fn test_merger_sum_aggr() { } struct MockConsumer { - input: BoxedExecutor, + input: Box, data: Arc>>, } @@ -259,7 +259,7 @@ impl StreamConsumer for MockConsumer { /// `SenderConsumer` consumes data from input executor and send it into a channel. pub struct SenderConsumer { - input: BoxedExecutor, + input: Box, channel: BoxedOutput, } diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs index 185d3fb5a6582..bf591f2a28f00 100644 --- a/src/stream/src/executor/lookup.rs +++ b/src/stream/src/executor/lookup.rs @@ -27,7 +27,7 @@ mod impl_; pub use impl_::LookupExecutorParams; -use super::{ActorContextRef, ExecutorInfo}; +use super::{ActorContextRef, Executor, ExecutorInfo}; #[cfg(test)] mod tests; @@ -53,10 +53,10 @@ pub struct LookupExecutor { stream: StreamJoinSide, /// The executor for arrangement. - arrangement_executor: Option>, + arrangement_executor: Option, /// The executor for stream. - stream_executor: Option>, + stream_executor: Option, /// The last received barrier. last_barrier: Option, @@ -83,10 +83,6 @@ pub struct LookupExecutor { #[async_trait] impl Execute for LookupExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/lookup/impl_.rs b/src/stream/src/executor/lookup/impl_.rs index 8f920f2dec853..d7878abc03022 100644 --- a/src/stream/src/executor/lookup/impl_.rs +++ b/src/stream/src/executor/lookup/impl_.rs @@ -36,7 +36,7 @@ use crate::executor::join::builder::JoinStreamChunkBuilder; use crate::executor::lookup::cache::LookupCache; use crate::executor::lookup::sides::{ArrangeJoinSide, ArrangeMessage, StreamJoinSide}; use crate::executor::lookup::LookupExecutor; -use crate::executor::{ActorContextRef, Barrier, Execute, ExecutorInfo, Message}; +use crate::executor::{ActorContextRef, Barrier, Executor, ExecutorInfo, Message}; use crate::task::AtomicU64Ref; /// Parameters for [`LookupExecutor`]. @@ -46,11 +46,11 @@ pub struct LookupExecutorParams { /// The side for arrangement. Currently, it should be a /// `MaterializeExecutor`. - pub arrangement: Box, + pub arrangement: Executor, /// The side for stream. It can be any stream, but it will generally be a /// `MaterializeExecutor`. - pub stream: Box, + pub stream: Executor, /// Should be the same as [`ColumnDesc`] in the arrangement. /// diff --git a/src/stream/src/executor/lookup/sides.rs b/src/stream/src/executor/lookup/sides.rs index abc972c76b8b8..df395418b7ed5 100644 --- a/src/stream/src/executor/lookup/sides.rs +++ b/src/stream/src/executor/lookup/sides.rs @@ -28,7 +28,7 @@ use risingwave_storage::table::batch_table::storage_table::StorageTable; use risingwave_storage::StateStore; use crate::executor::error::StreamExecutorError; -use crate::executor::{Barrier, BoxedMessageStream, Execute, Message, MessageStream}; +use crate::executor::{Barrier, BoxedMessageStream, Execute, Executor, Message, MessageStream}; /// Join side of Lookup Executor's stream pub(crate) struct StreamJoinSide { @@ -211,10 +211,7 @@ pub async fn align_barrier(mut left: BoxedMessageStream, mut right: BoxedMessage /// * Barrier (prev = `[2`], current = `[3`]) /// * `[Msg`] Arrangement (batch) #[try_stream(ok = ArrangeMessage, error = StreamExecutorError)] -pub async fn stream_lookup_arrange_prev_epoch( - stream: Box, - arrangement: Box, -) { +pub async fn stream_lookup_arrange_prev_epoch(stream: Executor, arrangement: Executor) { let mut input = pin!(align_barrier(stream.execute(), arrangement.execute())); let mut arrange_buf = vec![]; let mut stream_side_end = false; @@ -295,10 +292,7 @@ pub async fn stream_lookup_arrange_prev_epoch( /// * `[Do`] lookup `a` in arrangement of epoch `[2`] (current epoch) /// * Barrier (prev = `[2`], current = `[3`]) #[try_stream(ok = ArrangeMessage, error = StreamExecutorError)] -pub async fn stream_lookup_arrange_this_epoch( - stream: Box, - arrangement: Box, -) { +pub async fn stream_lookup_arrange_this_epoch(stream: Executor, arrangement: Executor) { let mut input = pin!(align_barrier(stream.execute(), arrangement.execute())); let mut stream_buf = vec![]; let mut arrange_buf = vec![]; diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index 5f2c203f6aac2..238319ee9e396 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -20,32 +20,30 @@ use futures_async_stream::try_stream; use itertools::Itertools; use super::error::StreamExecutorError; -use super::*; -use crate::executor::{BoxedMessageStream, ExecutorInfo}; +use super::{Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message}; /// Merges data from multiple inputs with order. If `order = [2, 1, 0]`, then /// it will first pipe data from the third input; after the third input gets a barrier, it will then /// pipe the second, and finally the first. In the future we could have more efficient /// implementation. pub struct LookupUnionExecutor { - info: ExecutorInfo, - inputs: Vec, + inputs: Vec, order: Vec, } -impl std::fmt::Debug for LookupUnionExecutor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LookupUnionExecutor") - .field("schema", &self.info.schema) - .field("pk_indices", &self.info.pk_indices) - .finish() - } -} +// TODO() +// impl std::fmt::Debug for LookupUnionExecutor { +// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +// f.debug_struct("LookupUnionExecutor") +// .field("schema", &self.info.schema) +// .field("pk_indices", &self.info.pk_indices) +// .finish() +// } +// } impl LookupUnionExecutor { - pub fn new(info: ExecutorInfo, inputs: Vec, order: Vec) -> Self { + pub fn new(inputs: Vec, order: Vec) -> Self { Self { - info, inputs, order: order.iter().map(|x| *x as _).collect(), } @@ -54,10 +52,6 @@ impl LookupUnionExecutor { #[async_trait] impl Execute for LookupUnionExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index e2105aabc236c..16097f1cd1ffb 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -37,9 +37,6 @@ pub struct MergeExecutor { /// The context of the actor. actor_context: ActorContextRef, - /// Logical Operator Info - info: ExecutorInfo, - /// Upstream channels. upstreams: Vec, @@ -60,7 +57,6 @@ impl MergeExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, fragment_id: FragmentId, upstream_fragment_id: FragmentId, inputs: Vec, @@ -70,7 +66,6 @@ impl MergeExecutor { ) -> Self { Self { actor_context: ctx, - info, upstreams: inputs, fragment_id, upstream_fragment_id, @@ -86,11 +81,6 @@ impl MergeExecutor { Self::new( ActorContext::for_test(114), - ExecutorInfo { - schema, - pk_indices: vec![], - identity: "MergeExecutor".to_string(), - }, 514, 1919, inputs @@ -245,10 +235,6 @@ impl MergeExecutor { } impl Execute for MergeExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index 5a8a46f9b3086..5c79e09891629 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -20,7 +20,6 @@ use await_tree::InstrumentAwait; use enum_as_inner::EnumAsInner; use futures::stream::BoxStream; use futures::{Stream, StreamExt}; -use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::StreamChunk; use risingwave_common::buffer::Bitmap; @@ -154,7 +153,6 @@ pub use wrapper::WrapperExecutor; use self::barrier_align::AlignedMessageStream; -pub type BoxedExecutor = Box; pub type MessageStreamItem = StreamExecutorResult; pub type BoxedMessageStream = BoxStream<'static, MessageStreamItem>; @@ -180,27 +178,13 @@ pub struct ExecutorInfo { /// `Executor` supports handling of control messages. pub trait Execute: Send + 'static { - fn info(&self) -> &ExecutorInfo; - - fn schema(&self) -> &Schema { - &self.info().schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info().pk_indices - } - - fn identity(&self) -> &str { - &self.info().identity - } - fn execute(self: Box) -> BoxedMessageStream; fn execute_with_epoch(self: Box, _epoch: u64) -> BoxedMessageStream { self.execute() } - fn boxed(self) -> BoxedExecutor + fn boxed(self) -> Box where Self: Sized + Send + 'static, { @@ -208,12 +192,62 @@ pub trait Execute: Send + 'static { } } -impl std::fmt::Debug for BoxedExecutor { +pub struct Executor { + info: ExecutorInfo, + execute: Box, +} + +impl Executor { + pub fn new(info: ExecutorInfo, execute: Box) -> Self { + Self { info, execute } + } + + pub fn info(&self) -> &ExecutorInfo { + &self.info + } + + pub fn schema(&self) -> &Schema { + &self.info.schema + } + + pub fn pk_indices(&self) -> PkIndicesRef<'_> { + &self.info.pk_indices + } + + pub fn identity(&self) -> &str { + &self.info.identity + } + + pub fn execute(self) -> BoxedMessageStream { + self.execute.execute() + } + + pub fn execute_with_epoch(self, epoch: u64) -> BoxedMessageStream { + self.execute.execute_with_epoch(epoch) + } +} + +impl std::fmt::Debug for Executor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.identity()) } } +impl From<(ExecutorInfo, Box)> for Executor { + fn from((info, execute): (ExecutorInfo, Box)) -> Self { + Self::new(info, execute) + } +} + +impl From<(ExecutorInfo, E)> for Executor +where + E: Execute, +{ + fn from((info, execute): (ExecutorInfo, E)) -> Self { + Self::new(info, execute.boxed()) + } +} + pub const INVALID_EPOCH: u64 = 0; type UpstreamFragmentId = FragmentId; diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index e354508fe3307..050c6e2beb3f9 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -24,7 +24,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::Bitmap; -use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, TableId}; +use risingwave_common::catalog::{ColumnDesc, ColumnId, ConflictBehavior, Schema, TableId}; use risingwave_common::row::{CompactedRow, RowDeserializer}; use risingwave_common::types::DataType; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; @@ -42,16 +42,16 @@ use crate::common::table::state_table::StateTableInner; use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContext, ActorContextRef, AddMutation, BoxedExecutor, - BoxedMessageStream, Execute, ExecutorInfo, Message, Mutation, StreamExecutorResult, - UpdateMutation, + expect_first_barrier, ActorContext, ActorContextRef, AddMutation, BoxedMessageStream, Execute, + Executor, Message, Mutation, StreamExecutorResult, UpdateMutation, }; use crate::task::{ActorId, AtomicU64Ref}; /// `MaterializeExecutor` materializes changes in stream into a materialized view on storage. pub struct MaterializeExecutor { - input: BoxedExecutor, - info: ExecutorInfo, + input: Executor, + + schema: Schema, state_table: StateTableInner, @@ -71,8 +71,8 @@ impl MaterializeExecutor { /// should be `None`. #[allow(clippy::too_many_arguments)] pub async fn new( - input: BoxedExecutor, - info: ExecutorInfo, + input: Executor, + schema: Schema, store: S, arrange_key: Vec, actor_context: ActorContextRef, @@ -99,7 +99,7 @@ impl MaterializeExecutor { Self { input, - info, + schema, state_table, arrange_key_indices, actor_context, @@ -115,7 +115,7 @@ impl MaterializeExecutor { let actor_id_str = self.actor_context.id.to_string(); let fragment_id_str = self.actor_context.fragment_id.to_string(); - let data_types = self.schema().data_types().clone(); + let data_types = self.schema.data_types(); let mut input = self.input.execute(); let barrier = expect_first_barrier(&mut input).await?; @@ -264,7 +264,8 @@ impl MaterializeExecutor { /// Create a new `MaterializeExecutor` without distribution info for test purpose. #[allow(clippy::too_many_arguments)] pub async fn for_test( - input: BoxedExecutor, + input: Executor, + schema: Schema, store: S, table_id: TableId, keys: Vec, @@ -293,14 +294,10 @@ impl MaterializeExecutor { Self { input, + schema, state_table, arrange_key_indices: arrange_columns.clone(), actor_context: ActorContext::for_test(0), - info: ExecutorInfo { - schema, - pk_indices: arrange_columns, - identity: format!("MaterializeExecutor {:X}", executor_id), - }, materialize_cache: MaterializeCache::new(watermark_epoch, MetricsInfo::for_test()), conflict_behavior, } @@ -430,19 +427,16 @@ impl MaterializeBuffer { } } impl Execute for MaterializeExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } } +// TODO() impl std::fmt::Debug for MaterializeExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("MaterializeExecutor") - .field("info", &self.info) + // .field("info", &self.info) .field("arrange_key_indices", &self.arrange_key_indices) .finish() } diff --git a/src/stream/src/executor/no_op.rs b/src/stream/src/executor/no_op.rs index fcc024f6b9a2a..ac12bf99d5d7f 100644 --- a/src/stream/src/executor/no_op.rs +++ b/src/stream/src/executor/no_op.rs @@ -12,31 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, ExecutorInfo}; +use super::{ActorContextRef, BoxedMessageStream, Execute, Executor}; /// No-op executor directly forwards the input stream. Currently used to break the multiple edges in /// the fragment graph. pub struct NoOpExecutor { _ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, } impl NoOpExecutor { - pub fn new(ctx: ActorContextRef, info: ExecutorInfo, input: BoxedExecutor) -> Self { - Self { - _ctx: ctx, - info, - input, - } + pub fn new(ctx: ActorContextRef, input: Executor) -> Self { + Self { _ctx: ctx, input } } } impl Execute for NoOpExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.input.execute() } diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index 6521b72d586cc..f04447a563329 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -25,13 +25,12 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ - Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, Mutation, StreamExecutorError, - Watermark, + Barrier, BoxedMessageStream, Execute, Message, Mutation, StreamExecutorError, Watermark, }; use crate::common::table::state_table::StateTable; pub struct NowExecutor { - info: ExecutorInfo, + data_types: Vec, /// Receiver of barrier channel. barrier_receiver: UnboundedReceiver, @@ -41,12 +40,12 @@ pub struct NowExecutor { impl NowExecutor { pub fn new( - info: ExecutorInfo, + data_types: Vec, barrier_receiver: UnboundedReceiver, state_table: StateTable, ) -> Self { Self { - info, + data_types, barrier_receiver, state_table, } @@ -55,10 +54,9 @@ impl NowExecutor { #[try_stream(ok = Message, error = StreamExecutorError)] async fn into_stream(self) { let Self { + data_types, barrier_receiver, mut state_table, - info, - .. } = self; // Whether the executor is paused. @@ -130,15 +128,12 @@ impl NowExecutor { let row = row::once(×tamp); state_table.update(last_row, row); - StreamChunk::from_rows( - &[(Op::Delete, last_row), (Op::Insert, row)], - &info.schema.data_types(), - ) + StreamChunk::from_rows(&[(Op::Delete, last_row), (Op::Insert, row)], &data_types) } else { let row = row::once(×tamp); state_table.insert(row); - StreamChunk::from_rows(&[(Op::Insert, row)], &info.schema.data_types()) + StreamChunk::from_rows(&[(Op::Insert, row)], &data_types) }; yield Message::Chunk(stream_chunk); @@ -155,10 +150,6 @@ impl NowExecutor { } impl Execute for NowExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 099bbade8a66c..720b57241e40f 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -20,6 +20,7 @@ use futures_async_stream::{for_await, try_stream}; use itertools::Itertools; use risingwave_common::array::stream_record::Record; use risingwave_common::array::{ArrayRef, Op, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::estimate_size::collections::EstimatedVecDeque; use risingwave_common::estimate_size::EstimateSize; use risingwave_common::row::{OwnedRow, Row, RowExt}; @@ -38,8 +39,8 @@ use crate::cache::{new_unbounded, ManagedLruCache}; use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, - ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, + Message, StreamExecutorError, StreamExecutorResult, }; use crate::task::AtomicU64Ref; @@ -94,14 +95,14 @@ type PartitionCache = ManagedLruCache; // TODO(rc): us /// - `WindowState` should output agg result for `curr output row`. /// - Recover: iterate through state table, push rows to `WindowState`, ignore ready windows. pub struct EowcOverWindowExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } struct ExecutorInner { actor_ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, calls: Vec, input_pk_indices: Vec, partition_key_indices: Vec, @@ -117,10 +118,6 @@ struct ExecutionVars { } impl Execute for EowcOverWindowExecutor { - fn info(&self) -> &ExecutorInfo { - &self.inner.info - } - fn execute(self: Box) -> BoxedMessageStream { self.executor_inner().boxed() } @@ -128,10 +125,10 @@ impl Execute for EowcOverWindowExecutor { pub struct EowcOverWindowExecutorArgs { pub actor_ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, + pub schema: Schema, pub calls: Vec, pub partition_key_indices: Vec, pub order_key_index: usize, @@ -147,7 +144,7 @@ impl EowcOverWindowExecutor { input: args.input, inner: ExecutorInner { actor_ctx: args.actor_ctx, - info: args.info, + schema: args.schema, calls: args.calls, input_pk_indices: input_info.pk_indices, partition_key_indices: args.partition_key_indices, @@ -228,7 +225,7 @@ impl EowcOverWindowExecutor { vars: &mut ExecutionVars, chunk: StreamChunk, ) -> StreamExecutorResult> { - let mut builders = this.info.schema.create_array_builders(chunk.capacity()); // just an estimate + let mut builders = this.schema.create_array_builders(chunk.capacity()); // just an estimate // We assume that the input is sorted by order key. for record in chunk.records() { @@ -309,7 +306,7 @@ impl EowcOverWindowExecutor { for key in keys_to_evict { let order_key = memcmp_encoding::decode_row( &key.order_key, - &[this.info.schema[this.order_key_index].data_type()], + &[this.schema[this.order_key_index].data_type()], &[OrderType::ascending()], )?; let state_row_pk = (&partition_key).chain(order_key).chain(key.pk); diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index 9aa0cb7649937..a7245c57f368c 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -23,6 +23,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::stream_record::Record; use risingwave_common::array::{Op, RowRef, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::row::{OwnedRow, Row, RowExt}; use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy; use risingwave_common::types::{DataType, DefaultOrdered}; @@ -46,8 +47,8 @@ use crate::common::StreamChunkBuilder; use crate::executor::monitor::StreamingMetrics; use crate::executor::over_window::over_partition::AffectedRange; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message, - StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, Execute, Executor, Message, StreamExecutorError, + StreamExecutorResult, }; use crate::task::AtomicU64Ref; @@ -57,14 +58,14 @@ use crate::task::AtomicU64Ref; /// - State table schema = output schema, state table pk = `partition key | order key | input pk`. /// - Output schema = input schema + window function results. pub struct OverWindowExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } struct ExecutorInner { actor_ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, calls: Vec, partition_key_indices: Vec, order_key_indices: Vec, @@ -99,10 +100,6 @@ struct ExecutionStats { } impl Execute for OverWindowExecutor { - fn info(&self) -> &ExecutorInfo { - &self.inner.info - } - fn execute(self: Box) -> crate::executor::BoxedMessageStream { self.executor_inner().boxed() } @@ -137,10 +134,10 @@ impl ExecutorInner { pub struct OverWindowExecutorArgs { pub actor_ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, + pub schema: Schema, pub calls: Vec, pub partition_key_indices: Vec, pub order_key_indices: Vec, @@ -187,7 +184,7 @@ impl OverWindowExecutor { input: args.input, inner: ExecutorInner { actor_ctx: args.actor_ctx, - info: args.info, + schema: args.schema, calls: args.calls, partition_key_indices: args.partition_key_indices, order_key_indices: args.order_key_indices, @@ -320,8 +317,7 @@ impl OverWindowExecutor { // `input pk` => `Record` let mut key_change_update_buffer: BTreeMap, Record> = BTreeMap::new(); - let mut chunk_builder = - StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types()); + let mut chunk_builder = StreamChunkBuilder::new(this.chunk_size, this.schema.data_types()); // Prepare things needed by metrics. let actor_id = this.actor_ctx.id.to_string(); diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index 8404bc5b9013d..edc012572c2dd 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -14,26 +14,30 @@ use std::fmt::{Debug, Formatter}; +use futures::StreamExt; +use futures_async_stream::try_stream; use multimap::MultiMap; use risingwave_common::array::StreamChunk; use risingwave_common::row::{Row, RowExt}; -use risingwave_common::types::ToOwnedDatum; +use risingwave_common::types::{ScalarImpl, ToOwnedDatum}; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; -use super::*; +use super::{ + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, StreamExecutorError, + StreamExecutorResult, Watermark, +}; /// `ProjectExecutor` project data with the `expr`. The `expr` takes a chunk of data, /// and returns a new data chunk. And then, `ProjectExecutor` will insert, delete /// or update element into next operator according to the result of the expression. pub struct ProjectExecutor { - input: BoxedExecutor, + input: Executor, inner: Inner, } struct Inner { _ctx: ActorContextRef, - info: ExecutorInfo, /// Expressions of the current projection. exprs: Vec, @@ -54,8 +58,7 @@ impl ProjectExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, + input: Executor, exprs: Vec, watermark_derivations: MultiMap, nondecreasing_expr_indices: Vec, @@ -66,7 +69,6 @@ impl ProjectExecutor { input, inner: Inner { _ctx: ctx, - info, exprs, watermark_derivations, nondecreasing_expr_indices, @@ -86,10 +88,6 @@ impl Debug for ProjectExecutor { } impl Execute for ProjectExecutor { - fn info(&self) -> &ExecutorInfo { - &self.inner.info - } - fn execute(self: Box) -> BoxedMessageStream { self.inner.execute(self.input).boxed() } @@ -133,8 +131,8 @@ impl Inner { ret.push(derived_watermark); } else { warn!( - "{} derive a NULL watermark with the expression {}!", - self.info.identity, out_col_idx + "a NULL watermark is derived with the expression {}!", + out_col_idx ); } } @@ -142,7 +140,7 @@ impl Inner { } #[try_stream(ok = Message, error = StreamExecutorError)] - async fn execute(mut self, input: BoxedExecutor) { + async fn execute(mut self, input: Executor) { #[for_await] for msg in input.execute() { let msg = msg?; diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs index ed327cb97abf0..9fadb5949dac2 100644 --- a/src/stream/src/executor/project_set.rs +++ b/src/stream/src/executor/project_set.rs @@ -27,9 +27,7 @@ use risingwave_expr::expr::{LogReport, NonStrictExpression}; use risingwave_expr::table_function::ProjectSetSelectItem; use super::error::StreamExecutorError; -use super::{ - ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message, StreamExecutorResult, Watermark, -}; +use super::{ActorContextRef, Execute, Executor, Message, StreamExecutorResult, Watermark}; use crate::common::StreamChunkBuilder; const PROJ_ROW_ID_OFFSET: usize = 1; @@ -38,13 +36,12 @@ const PROJ_ROW_ID_OFFSET: usize = 1; /// and returns a new data chunk. And then, `ProjectSetExecutor` will insert, delete /// or update element into next operator according to the result of the expression. pub struct ProjectSetExecutor { - input: BoxedExecutor, + input: Executor, inner: Inner, } struct Inner { _ctx: ActorContextRef, - info: ExecutorInfo, /// Expressions of the current project_section. select_list: Vec, @@ -60,8 +57,7 @@ impl ProjectSetExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, + input: Executor, select_list: Vec, chunk_size: usize, watermark_derivations: MultiMap, @@ -69,7 +65,6 @@ impl ProjectSetExecutor { ) -> Self { let inner = Inner { _ctx: ctx, - info, select_list, chunk_size, watermark_derivations, @@ -89,10 +84,6 @@ impl Debug for ProjectSetExecutor { } impl Execute for ProjectSetExecutor { - fn info(&self) -> &ExecutorInfo { - &self.inner.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.inner.execute(self.input).boxed() } @@ -100,7 +91,7 @@ impl Execute for ProjectSetExecutor { impl Inner { #[try_stream(ok = Message, error = StreamExecutorError)] - async fn execute(self, input: BoxedExecutor) { + async fn execute(self, input: Executor) { assert!(!self.select_list.is_empty()); // First column will be `projected_row_id`, which represents the index in the // output table @@ -260,8 +251,8 @@ impl Inner { ret.push(derived_watermark); } else { warn!( - "{} derive a NULL watermark with the expression {}!", - self.info.identity, expr_idx + "a NULL watermark is derived with the expression {}!", + expr_idx ); } } diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs index f56426778c394..0b51bab665d41 100644 --- a/src/stream/src/executor/rearranged_chain.rs +++ b/src/stream/src/executor/rearranged_chain.rs @@ -21,9 +21,7 @@ use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; use super::error::StreamExecutorError; -use super::{ - expect_first_barrier, Barrier, BoxedExecutor, Execute, ExecutorInfo, Message, MessageStream, -}; +use super::{expect_first_barrier, Barrier, Execute, Executor, Message, MessageStream}; use crate::task::{ActorId, CreateMviewProgress}; /// `ChainExecutor` is an executor that enables synchronization between the existing stream and @@ -34,11 +32,9 @@ use crate::task::{ActorId, CreateMviewProgress}; /// [`RearrangedChainExecutor`] resolves the latency problem when creating MV with a huge amount of /// existing data, by rearranging the barrier from the upstream. Check the design doc for details. pub struct RearrangedChainExecutor { - info: ExecutorInfo, + snapshot: Executor, - snapshot: BoxedExecutor, - - upstream: BoxedExecutor, + upstream: Executor, progress: CreateMviewProgress, @@ -83,14 +79,8 @@ impl RearrangedMessage { } impl RearrangedChainExecutor { - pub fn new( - info: ExecutorInfo, - snapshot: BoxedExecutor, - upstream: BoxedExecutor, - progress: CreateMviewProgress, - ) -> Self { + pub fn new(snapshot: Executor, upstream: Executor, progress: CreateMviewProgress) -> Self { Self { - info, snapshot, upstream, actor_id: progress.actor_id(), @@ -288,10 +278,6 @@ impl RearrangedChainExecutor { } impl Execute for RearrangedChainExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index 513e71ddd5321..8df2d988fca7d 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -30,9 +30,6 @@ use crate::task::{FragmentId, SharedContext}; /// there should be a `ReceiverExecutor` running in the background, so as to push /// messages down to the executors. pub struct ReceiverExecutor { - /// Logical Operator Info - info: ExecutorInfo, - /// Input from upstream. input: BoxedInput, @@ -52,11 +49,12 @@ pub struct ReceiverExecutor { metrics: Arc, } +// TODO() impl std::fmt::Debug for ReceiverExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ReceiverExecutor") - .field("schema", &self.info.schema) - .field("pk_indices", &self.info.pk_indices) + // .field("schema", &self.info.schema) + // .field("pk_indices", &self.info.pk_indices) .finish() } } @@ -65,7 +63,6 @@ impl ReceiverExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, fragment_id: FragmentId, upstream_fragment_id: FragmentId, input: BoxedInput, @@ -75,7 +72,6 @@ impl ReceiverExecutor { ) -> Self { Self { input, - info, actor_context: ctx, upstream_fragment_id, metrics, @@ -86,19 +82,12 @@ impl ReceiverExecutor { #[cfg(test)] pub fn for_test(input: super::exchange::permit::Receiver) -> Self { - use risingwave_common::catalog::Schema; - use super::exchange::input::LocalInput; use crate::executor::exchange::input::Input; use crate::executor::ActorContext; Self::new( ActorContext::for_test(114), - ExecutorInfo { - schema: Schema::default(), - pk_indices: vec![], - identity: "ReceiverExecutor".to_string(), - }, 514, 1919, LocalInput::new(input, 0).boxed_input(), @@ -110,10 +99,6 @@ impl ReceiverExecutor { } impl Execute for ReceiverExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(mut self: Box) -> BoxedMessageStream { let actor_id = self.actor_context.id; diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index dd52aad8cd86d..d645221bdbcf5 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -24,15 +24,14 @@ use risingwave_common::types::Serial; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::row_id::RowIdGenerator; -use super::{expect_first_barrier, ActorContextRef, BoxedExecutor, Execute, ExecutorInfo}; +use super::{expect_first_barrier, ActorContextRef, Execute, Executor}; use crate::executor::{Message, StreamExecutorError}; /// [`RowIdGenExecutor`] generates row id for data, where the user has not specified a pk. pub struct RowIdGenExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - upstream: Option, + upstream: Option, row_id_index: usize, @@ -42,14 +41,12 @@ pub struct RowIdGenExecutor { impl RowIdGenExecutor { pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - upstream: BoxedExecutor, + upstream: Executor, row_id_index: usize, vnodes: Bitmap, ) -> Self { Self { ctx, - info, upstream: Some(upstream), row_id_index, row_id_generator: Self::new_generator(&vnodes), @@ -124,10 +121,6 @@ impl RowIdGenExecutor { } impl Execute for RowIdGenExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index ee9e4b8b7647d..90e8244aaa9b4 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -47,7 +47,7 @@ use crate::task::AtomicU64Ref; /// Therefore, we "automatically" implemented a window function inside /// `SimpleAggExecutor`. pub struct SimpleAggExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } @@ -112,10 +112,6 @@ struct ExecutionVars { } impl Execute for SimpleAggExecutor { - fn info(&self) -> &ExecutorInfo { - &self.inner.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index bd169336e7d13..8e81006e4133a 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -34,7 +34,7 @@ use risingwave_connector::sink::{ use thiserror_ext::AsReport; use super::error::{StreamExecutorError, StreamExecutorResult}; -use super::{BoxedExecutor, Execute, ExecutorInfo, Message, PkIndices}; +use super::{Execute, Executor, ExecutorInfo, Message, PkIndices}; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedMessageStream, MessageStream, Mutation, }; @@ -43,7 +43,7 @@ use crate::task::ActorId; pub struct SinkExecutor { actor_context: ActorContextRef, info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, sink: SinkImpl, input_columns: Vec, sink_param: SinkParam, @@ -83,7 +83,7 @@ impl SinkExecutor { pub async fn new( actor_context: ActorContextRef, info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, sink_writer_param: SinkWriterParam, sink_param: SinkParam, columns: Vec, @@ -413,10 +413,6 @@ impl SinkExecutor { } impl Execute for SinkExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner() } diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index 54672aecc255c..0153a7e7c7dd6 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -15,27 +15,28 @@ use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::Op; +use risingwave_common::catalog::Schema; use risingwave_storage::StateStore; use super::sort_buffer::SortBuffer; use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, - ExecutorInfo, Message, StreamExecutorError, Watermark, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, Watermark, }; use crate::common::table::state_table::StateTable; use crate::common::StreamChunkBuilder; pub struct SortExecutor { - input: BoxedExecutor, + input: Executor, inner: ExecutorInner, } pub struct SortExecutorArgs { pub actor_ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, + pub schema: Schema, pub buffer_table: StateTable, pub chunk_size: usize, pub sort_column_index: usize, @@ -43,8 +44,8 @@ pub struct SortExecutorArgs { struct ExecutorInner { actor_ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, buffer_table: StateTable, chunk_size: usize, sort_column_index: usize, @@ -55,10 +56,6 @@ struct ExecutionVars { } impl Execute for SortExecutor { - fn info(&self) -> &ExecutorInfo { - &self.inner.info - } - fn execute(self: Box) -> BoxedMessageStream { self.executor_inner().boxed() } @@ -70,7 +67,7 @@ impl SortExecutor { input: args.input, inner: ExecutorInner { actor_ctx: args.actor_ctx, - info: args.info, + schema: args.schema, buffer_table: args.buffer_table, chunk_size: args.chunk_size, sort_column_index: args.sort_column_index, @@ -105,7 +102,7 @@ impl SortExecutor { if col_idx == this.sort_column_index => { let mut chunk_builder = - StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types()); + StreamChunkBuilder::new(this.chunk_size, this.schema.data_types()); #[for_await] for row in vars diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs index 42e81880fcd0e..da8ad80723576 100644 --- a/src/stream/src/executor/source/fetch_executor.rs +++ b/src/stream/src/executor/source/fetch_executor.rs @@ -39,8 +39,13 @@ use risingwave_storage::store::PrefetchOptions; use risingwave_storage::StateStore; use thiserror_ext::AsReport; +use super::{get_split_offset_col_idx, SourceStateTableHandler}; use crate::executor::stream_reader::StreamReaderWithPause; -use crate::executor::*; +use crate::executor::{ + expect_first_barrier, get_split_offset_mapping_from_chunk, prune_additional_cols, + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, Mutation, StreamExecutorError, + StreamExecutorResult, StreamSourceCore, +}; const SPLIT_BATCH_SIZE: usize = 1000; @@ -48,13 +53,12 @@ type SplitBatch = Option>; pub struct FsFetchExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: Option>, /// Upstream list executor. - upstream: Option, + upstream: Option, // control options for connector level source_ctrl_opts: SourceCtrlOpts, @@ -69,15 +73,13 @@ impl FsFetchExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: StreamSourceCore, - upstream: BoxedExecutor, + upstream: Executor, source_ctrl_opts: SourceCtrlOpts, connector_params: ConnectorParams, ) -> Self { Self { actor_ctx, - info, stream_source_core: Some(stream_source_core), upstream: Some(upstream), source_ctrl_opts, @@ -355,22 +357,19 @@ impl FsFetchExecutor { } impl Execute for FsFetchExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } } +// TODO() impl Debug for FsFetchExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if let Some(core) = &self.stream_source_core { f.debug_struct("FsFetchExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - .field("pk_indices", &self.info.pk_indices) + // .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("FsFetchExecutor").finish() diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index 884e4446ae033..d084b7c506534 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -48,7 +48,6 @@ const WAIT_BARRIER_MULTIPLE_TIMES: u128 = 5; /// such as s3. pub struct FsSourceExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: StreamSourceCore, @@ -69,7 +68,6 @@ impl FsSourceExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: StreamSourceCore, metrics: Arc, barrier_receiver: UnboundedReceiver, @@ -78,7 +76,6 @@ impl FsSourceExecutor { ) -> StreamResult { Ok(Self { actor_ctx, - info, stream_source_core, metrics, barrier_receiver: Some(barrier_receiver), @@ -479,21 +476,18 @@ impl FsSourceExecutor { } impl Execute for FsSourceExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } } +// TODO() impl Debug for FsSourceExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("FsSourceExecutor") .field("source_id", &self.stream_source_core.source_id) .field("column_ids", &self.stream_source_core.column_ids) - .field("pk_indices", &self.info.pk_indices) + // .field("pk_indices", &self.info.pk_indices) .finish() } } diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs index a96b2dc392e50..f13dfdd949f0b 100644 --- a/src/stream/src/executor/source/list_executor.rs +++ b/src/stream/src/executor/source/list_executor.rs @@ -38,7 +38,6 @@ const CHUNK_SIZE: usize = 1024; #[allow(dead_code)] pub struct FsListExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: Option>, @@ -63,7 +62,6 @@ impl FsListExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: Option>, metrics: Arc, barrier_receiver: UnboundedReceiver, @@ -73,7 +71,6 @@ impl FsListExecutor { ) -> Self { Self { actor_ctx, - info, stream_source_core, metrics, barrier_receiver: Some(barrier_receiver), @@ -193,22 +190,19 @@ impl FsListExecutor { } impl Execute for FsListExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } } +// TODO() impl Debug for FsListExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if let Some(core) = &self.stream_source_core { f.debug_struct("FsListExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - .field("pk_indices", &self.info.pk_indices) + // .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("FsListExecutor").finish() diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index d040673ce29d7..9faf9f324efc9 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -43,7 +43,6 @@ const WAIT_BARRIER_MULTIPLE_TIMES: u128 = 5; pub struct SourceExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: Option>, @@ -68,7 +67,6 @@ impl SourceExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: Option>, metrics: Arc, barrier_receiver: UnboundedReceiver, @@ -78,7 +76,6 @@ impl SourceExecutor { ) -> Self { Self { actor_ctx, - info, stream_source_core, metrics, barrier_receiver: Some(barrier_receiver), @@ -554,8 +551,7 @@ impl SourceExecutor { // chunks. self_paused = true; tracing::warn!( - "source {} paused, wait barrier for {:?}", - self.info.identity, + "source paused, wait barrier for {:?}", last_barrier_time.elapsed() ); stream.pause_stream(); @@ -643,10 +639,6 @@ impl SourceExecutor { } impl Execute for SourceExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { if self.stream_source_core.is_some() { self.execute_with_stream_source().boxed() @@ -656,13 +648,14 @@ impl Execute for SourceExecutor { } } +// TODO() impl Debug for SourceExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if let Some(core) = &self.stream_source_core { f.debug_struct("SourceExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - .field("pk_indices", &self.info.pk_indices) + // .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("SourceExecutor").finish() diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index 86862f2f3c41d..586f2e0003f52 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -28,17 +28,13 @@ use crate::error::StreamResult; pub struct StatelessSimpleAggExecutor { _ctx: ActorContextRef, - pub(super) info: ExecutorInfo, - pub(super) input: Box, + pub(super) input: Executor, + pub(super) schema: Schema, pub(super) aggs: Vec, pub(super) agg_calls: Vec, } impl Execute for StatelessSimpleAggExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } @@ -64,7 +60,7 @@ impl StatelessSimpleAggExecutor { let StatelessSimpleAggExecutor { _ctx, input, - info, + schema, aggs, agg_calls, } = self; @@ -85,7 +81,7 @@ impl StatelessSimpleAggExecutor { if is_dirty { is_dirty = false; - let mut builders = info.schema.create_array_builders(1); + let mut builders = schema.create_array_builders(1); for ((agg, state), builder) in aggs .iter() .zip_eq_fast(states.iter_mut()) @@ -115,15 +111,15 @@ impl StatelessSimpleAggExecutor { impl StatelessSimpleAggExecutor { pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, + input: Executor, + schema: Schema, agg_calls: Vec, ) -> StreamResult { let aggs = agg_calls.iter().map(build_retractable).try_collect()?; Ok(StatelessSimpleAggExecutor { _ctx: ctx, - info, input, + schema, aggs, agg_calls, }) diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs index 3820d6a5bea42..a6e18e2adbc27 100644 --- a/src/stream/src/executor/subscription.rs +++ b/src/stream/src/executor/subscription.rs @@ -22,8 +22,8 @@ use risingwave_storage::store::LocalStateStore; use tokio::time::Instant; use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, - ExecutorInfo, Message, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, + Message, StreamExecutorError, StreamExecutorResult, }; use crate::common::log_store_impl::kv_log_store::ReaderTruncationOffsetType; use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter; @@ -32,8 +32,7 @@ const EXECUTE_GC_INTERVAL: u64 = 3600; pub struct SubscriptionExecutor { actor_context: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, log_store: SubscriptionLogStoreWriter, retention_seconds: u64, } @@ -43,14 +42,12 @@ impl SubscriptionExecutor { #[expect(clippy::unused_async)] pub async fn new( actor_context: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, log_store: SubscriptionLogStoreWriter, retention_seconds: u64, ) -> StreamExecutorResult { Ok(Self { actor_context, - info, input, log_store, retention_seconds, @@ -115,10 +112,6 @@ impl SubscriptionExecutor { } } impl Execute for SubscriptionExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index b7086239ad1da..f17851696ee38 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -20,7 +20,7 @@ use tokio::sync::mpsc::error::SendError; use tokio_stream::wrappers::ReceiverStream; use super::actor::spawn_blocking_drop_stream; -use super::{BoxedExecutor, Execute, ExecutorInfo, Message, MessageStreamItem}; +use super::{Execute, Executor, ExecutorInfo, Message, MessageStreamItem}; use crate::task::ActorId; /// Handle used to drive the subtask. @@ -35,10 +35,6 @@ pub struct SubtaskRxExecutor { } impl Execute for SubtaskRxExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { ReceiverStream::new(self.rx).boxed() } @@ -50,7 +46,7 @@ impl Execute for SubtaskRxExecutor { /// Used when there're multiple stateful executors in an actor. These subtasks can be concurrently /// executed to improve the I/O performance, while the computing resource can be still bounded to a /// single thread. -pub fn wrap(input: BoxedExecutor, actor_id: ActorId) -> (SubtaskHandle, SubtaskRxExecutor) { +pub fn wrap(input: Executor, actor_id: ActorId) -> (SubtaskHandle, SubtaskRxExecutor) { let (tx, rx) = mpsc::channel(1); let rx_executor = SubtaskRxExecutor { info: ExecutorInfo { diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs index 9ce0e41367758..a671eb5c0fc5b 100644 --- a/src/stream/src/executor/temporal_join.rs +++ b/src/stream/src/executor/temporal_join.rs @@ -47,14 +47,14 @@ use crate::cache::{cache_may_stale, new_with_hasher_in, ManagedLruCache}; use crate::common::metrics::MetricsInfo; use crate::executor::join::builder::JoinStreamChunkBuilder; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{ActorContextRef, BoxedExecutor, Watermark}; +use crate::executor::{ActorContextRef, Executor, Watermark}; use crate::task::AtomicU64Ref; pub struct TemporalJoinExecutor { ctx: ActorContextRef, info: ExecutorInfo, - left: BoxedExecutor, - right: BoxedExecutor, + left: Executor, + right: Executor, right_table: TemporalSide, left_join_keys: Vec, right_join_keys: Vec, @@ -280,7 +280,7 @@ async fn internal_messages_until_barrier(stream: impl MessageStream, expected_ba // any number of `InternalMessage::Chunk(left_chunk)` and followed by // `InternalMessage::Barrier(right_chunks, barrier)`. #[try_stream(ok = InternalMessage, error = StreamExecutorError)] -async fn align_input(left: Box, right: Box) { +async fn align_input(left: Executor, right: Executor) { let mut left = pin!(left.execute()); let mut right = pin!(right.execute()); // Keep producing intervals until stream exhaustion or errors. @@ -332,8 +332,8 @@ impl TemporalJoinExecutor pub fn new( ctx: ActorContextRef, info: ExecutorInfo, - left: BoxedExecutor, - right: BoxedExecutor, + left: Executor, + right: Executor, table: StorageTable, left_join_keys: Vec, right_join_keys: Vec, @@ -491,10 +491,6 @@ impl TemporalJoinExecutor impl Execute for TemporalJoinExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.into_stream().boxed() } diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index fa8c53482ad19..e76ae8c391b79 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -180,10 +180,6 @@ impl MockSource { } impl Execute for MockSource { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } @@ -298,14 +294,14 @@ pub mod agg_executor { }; use crate::executor::aggregation::AggStateStorage; use crate::executor::{ - ActorContext, ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, HashAggExecutor, - PkIndices, SimpleAggExecutor, + ActorContext, ActorContextRef, Execute, Executor, ExecutorInfo, HashAggExecutor, PkIndices, + SimpleAggExecutor, }; /// Generate aggExecuter's schema from `input`, `agg_calls` and `group_key_indices`. /// For [`crate::executor::HashAggExecutor`], the group key indices should be provided. pub fn generate_agg_schema( - input: &dyn Execute, + input_ref: &Executor, agg_calls: &[AggCall], group_key_indices: Option<&[usize]>, ) -> Schema { @@ -316,7 +312,7 @@ pub mod agg_executor { let fields = if let Some(key_indices) = group_key_indices { let keys = key_indices .iter() - .map(|idx| input.schema().fields[*idx].clone()); + .map(|idx| input_ref.schema().fields[*idx].clone()); keys.chain(aggs).collect() } else { @@ -334,7 +330,7 @@ pub mod agg_executor { agg_call: &AggCall, group_key_indices: &[usize], pk_indices: &[usize], - input_ref: &dyn Execute, + input_ref: &Executor, is_append_only: bool, ) -> AggStateStorage { match agg_call.kind { @@ -405,7 +401,7 @@ pub mod agg_executor { table_id: TableId, agg_calls: &[AggCall], group_key_indices: &[usize], - input_ref: &dyn Execute, + input_ref: &Executor, ) -> StateTable { let input_fields = input_ref.schema().fields(); @@ -444,7 +440,7 @@ pub mod agg_executor { #[allow(clippy::too_many_arguments)] pub async fn new_boxed_hash_agg_executor( store: S, - input: Box, + input: Executor, is_append_only: bool, agg_calls: Vec, row_count_index: usize, @@ -453,7 +449,7 @@ pub mod agg_executor { extreme_cache_size: usize, emit_on_window_close: bool, executor_id: u64, - ) -> Box { + ) -> Executor { let mut storages = Vec::with_capacity(agg_calls.iter().len()); for (idx, agg_call) in agg_calls.iter().enumerate() { storages.push( @@ -463,7 +459,7 @@ pub mod agg_executor { agg_call, &group_key_indices, &pk_indices, - input.as_ref(), + &input, is_append_only, ) .await, @@ -475,23 +471,23 @@ pub mod agg_executor { TableId::new(agg_calls.len() as u32), &agg_calls, &group_key_indices, - input.as_ref(), + &input, ) .await; - let schema = generate_agg_schema(input.as_ref(), &agg_calls, Some(&group_key_indices)); + let schema = generate_agg_schema(&input, &agg_calls, Some(&group_key_indices)); let info = ExecutorInfo { schema, pk_indices, identity: format!("HashAggExecutor {:X}", executor_id), }; - HashAggExecutor::::new(AggExecutorArgs { + let exec = HashAggExecutor::::new(AggExecutorArgs { version: PbAggNodeVersion::Max, input, actor_ctx: ActorContext::for_test(123), - info, + info: info.clone(), extreme_cache_size, @@ -509,21 +505,21 @@ pub mod agg_executor { emit_on_window_close, }, }) - .unwrap() - .boxed() + .unwrap(); + (info, exec).into() } #[allow(clippy::too_many_arguments)] pub async fn new_boxed_simple_agg_executor( actor_ctx: ActorContextRef, store: S, - input: BoxedExecutor, + input: Executor, is_append_only: bool, agg_calls: Vec, row_count_index: usize, pk_indices: PkIndices, executor_id: u64, - ) -> Box { + ) -> Executor { let storages = future::join_all(agg_calls.iter().enumerate().map(|(idx, agg_call)| { create_agg_state_storage( store.clone(), @@ -531,7 +527,7 @@ pub mod agg_executor { agg_call, &[], &pk_indices, - input.as_ref(), + &input, is_append_only, ) })) @@ -542,23 +538,23 @@ pub mod agg_executor { TableId::new(agg_calls.len() as u32), &agg_calls, &[], - input.as_ref(), + &input, ) .await; - let schema = generate_agg_schema(input.as_ref(), &agg_calls, None); + let schema = generate_agg_schema(&input, &agg_calls, None); let info = ExecutorInfo { schema, pk_indices, identity: format!("SimpleAggExecutor {:X}", executor_id), }; - SimpleAggExecutor::new(AggExecutorArgs { + let exec = SimpleAggExecutor::new(AggExecutorArgs { version: PbAggNodeVersion::Max, input, actor_ctx, - info, + info: info.clone(), extreme_cache_size: 1024, @@ -570,8 +566,8 @@ pub mod agg_executor { watermark_epoch: Arc::new(AtomicU64::new(0)), extra: SimpleAggExecutorExtraArgs {}, }) - .unwrap() - .boxed() + .unwrap(); + (info, exec).into() } } diff --git a/src/stream/src/executor/top_n/group_top_n.rs b/src/stream/src/executor/top_n/group_top_n.rs index eb9889abf25fb..11e29d7ddfc72 100644 --- a/src/stream/src/executor/top_n/group_top_n.rs +++ b/src/stream/src/executor/top_n/group_top_n.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::Bitmap; +use risingwave_common::catalog::Schema; use risingwave_common::hash::HashKey; use risingwave_common::row::RowExt; use risingwave_common::util::epoch::EpochPair; @@ -32,7 +33,7 @@ use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; use crate::task::AtomicU64Ref; pub type GroupTopNExecutor = @@ -41,9 +42,9 @@ pub type GroupTopNExecutor = impl GroupTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -55,7 +56,7 @@ impl GroupTopNExecutor GroupTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// `LIMIT XXX`. None means no limit. limit: usize, @@ -97,7 +98,7 @@ pub struct InnerGroupTopNExecutor InnerGroupTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -113,12 +114,11 @@ impl InnerGroupTopNExecutor::new(state_table, cache_key_serde.clone()); Ok(Self { - info, + schema, offset: offset_and_limit.0, limit: offset_and_limit.1, managed_state, @@ -191,7 +191,7 @@ where .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) .inc(); let mut topn_cache = - TopNCache::new(self.offset, self.limit, self.info().schema.data_types()); + TopNCache::new(self.offset, self.limit, self.schema.data_types()); self.managed_state .init_topn_cache(Some(group_key), &mut topn_cache) .await?; @@ -227,7 +227,7 @@ where .group_top_n_cached_entry_count .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) .set(self.caches.len() as i64); - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -238,10 +238,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn update_epoch(&mut self, epoch: u64) { self.caches.update_epoch(epoch); } diff --git a/src/stream/src/executor/top_n/group_top_n_appendonly.rs b/src/stream/src/executor/top_n/group_top_n_appendonly.rs index edbe54a7ff7ac..94f1df73770ab 100644 --- a/src/stream/src/executor/top_n/group_top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/group_top_n_appendonly.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::Bitmap; +use risingwave_common::catalog::Schema; use risingwave_common::hash::HashKey; use risingwave_common::row::{RowDeserializer, RowExt}; use risingwave_common::util::epoch::EpochPair; @@ -31,7 +32,7 @@ use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; use crate::task::AtomicU64Ref; /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need @@ -45,9 +46,9 @@ impl { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -59,7 +60,7 @@ impl input, ctx: ctx.clone(), inner: InnerAppendOnlyGroupTopNExecutor::new( - info, + schema, storage_key, offset_and_limit, order_by, @@ -73,7 +74,7 @@ impl } pub struct InnerAppendOnlyGroupTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// `LIMIT XXX`. None means no limit. limit: usize, @@ -103,7 +104,7 @@ impl { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -119,12 +120,11 @@ impl "GroupTopN", ); - let cache_key_serde = - create_cache_key_serde(&storage_key, &info.schema, &order_by, &group_by); + let cache_key_serde = create_cache_key_serde(&storage_key, &schema, &order_by, &group_by); let managed_state = ManagedTopNState::::new(state_table, cache_key_serde.clone()); Ok(Self { - info, + schema, offset: offset_and_limit.0, limit: offset_and_limit.1, managed_state, @@ -147,7 +147,7 @@ where let mut res_rows = Vec::with_capacity(self.limit); let keys = K::build(&self.group_by, chunk.data_chunk())?; - let data_types = self.info().schema.data_types(); + let data_types = self.schema.data_types(); let row_deserializer = RowDeserializer::new(data_types.clone()); let table_id_str = self.managed_state.table().table_id().to_string(); let actor_id_str = self.ctx.id.to_string(); @@ -197,7 +197,7 @@ where .group_top_n_appendonly_cached_entry_count .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) .set(self.caches.len() as i64); - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -208,10 +208,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc) { let cache_may_stale = self.managed_state.update_vnode_bitmap(vnode_bitmap); if cache_may_stale { diff --git a/src/stream/src/executor/top_n/top_n_appendonly.rs b/src/stream/src/executor/top_n/top_n_appendonly.rs index 7fce9c7adfd1d..f1a477344c90a 100644 --- a/src/stream/src/executor/top_n/top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/top_n_appendonly.rs @@ -13,6 +13,7 @@ // limitations under the License. use risingwave_common::array::{Op, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::row::{RowDeserializer, RowExt}; use risingwave_common::util::epoch::EpochPair; use risingwave_common::util::sort_util::ColumnOrder; @@ -24,7 +25,7 @@ use super::{ManagedTopNState, TopNCache, NO_GROUP_KEY}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Execute, Executor, PkIndices, Watermark}; /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need /// to keep all the rows seen. As long as a record @@ -38,9 +39,9 @@ pub type AppendOnlyTopNExecutor = impl AppendOnlyTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -50,7 +51,7 @@ impl AppendOnlyTopNExecutor input, ctx, inner: InnerAppendOnlyTopNExecutor::new( - info, + schema, storage_key, offset_and_limit, order_by, @@ -61,7 +62,7 @@ impl AppendOnlyTopNExecutor } pub struct InnerAppendOnlyTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// The storage key indices of the `TopNExecutor` storage_key_indices: PkIndices, @@ -80,7 +81,7 @@ pub struct InnerAppendOnlyTopNExecutor { impl InnerAppendOnlyTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -89,12 +90,12 @@ impl InnerAppendOnlyTopNExecutor::new(state_table, cache_key_serde.clone()); - let data_types = info.schema.data_types(); + let data_types = schema.data_types(); Ok(Self { - info, + schema, managed_state, storage_key_indices: storage_key.into_iter().map(|op| op.column_index).collect(), cache: TopNCache::new(num_offset, num_limit, data_types), @@ -111,7 +112,7 @@ where async fn apply_chunk(&mut self, chunk: StreamChunk) -> StreamExecutorResult { let mut res_ops = Vec::with_capacity(self.cache.limit); let mut res_rows = Vec::with_capacity(self.cache.limit); - let data_types = self.info().schema.data_types(); + let data_types = self.schema.data_types(); let row_deserializer = RowDeserializer::new(data_types); // apply the chunk to state table for (op, row_ref) in chunk.rows() { @@ -128,7 +129,7 @@ where )?; } - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -139,10 +140,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - async fn init(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { self.managed_state.init_epoch(epoch); self.managed_state diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs index b053985a2859d..7ff65ea64183d 100644 --- a/src/stream/src/executor/top_n/top_n_plain.rs +++ b/src/stream/src/executor/top_n/top_n_plain.rs @@ -13,6 +13,7 @@ // limitations under the License. use risingwave_common::array::{Op, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::row::RowExt; use risingwave_common::util::epoch::EpochPair; use risingwave_common::util::sort_util::ColumnOrder; @@ -23,7 +24,7 @@ use super::{ManagedTopNState, TopNCache, TopNCacheTrait}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Execute, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Execute, Executor, PkIndices, Watermark}; /// `TopNExecutor` works with input with modification, it keeps all the data /// records/rows that have been seen, and returns topN records overall. @@ -33,9 +34,9 @@ pub type TopNExecutor = impl TopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -45,7 +46,7 @@ impl TopNExecutor { input, ctx, inner: InnerTopNExecutor::new( - info, + schema, storage_key, offset_and_limit, order_by, @@ -79,7 +80,7 @@ impl TopNExecutor { } pub struct InnerTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// The storage key indices of the `TopNExecutor` storage_key_indices: PkIndices, @@ -103,7 +104,7 @@ impl InnerTopNExecutor { /// into `CacheKey`. #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -112,12 +113,12 @@ impl InnerTopNExecutor { let num_offset = offset_and_limit.0; let num_limit = offset_and_limit.1; - let cache_key_serde = create_cache_key_serde(&storage_key, &info.schema, &order_by, &[]); + let cache_key_serde = create_cache_key_serde(&storage_key, &schema, &order_by, &[]); let managed_state = ManagedTopNState::::new(state_table, cache_key_serde.clone()); - let data_types = info.schema.data_types(); + let data_types = schema.data_types(); Ok(Self { - info, + schema, managed_state, storage_key_indices: storage_key.into_iter().map(|op| op.column_index).collect(), cache: TopNCache::new(num_offset, num_limit, data_types), @@ -162,7 +163,7 @@ where } } } - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -173,10 +174,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - async fn init(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { self.managed_state.init_epoch(epoch); self.managed_state diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index aad7e23b53169..2f238af2e8995 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -30,8 +30,8 @@ use risingwave_common::util::sort_util::ColumnOrder; use super::CacheKey; use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Execute, - ExecutorInfo, Message, Watermark, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, + Message, Watermark, }; pub trait TopNExecutorBase: Send + 'static { @@ -50,8 +50,6 @@ pub trait TopNExecutorBase: Send + 'static { /// Flush the buffered chunk to the storage backend. fn try_flush_data(&mut self) -> impl Future> + Send; - fn info(&self) -> &ExecutorInfo; - /// Update the vnode bitmap for the state table and manipulate the cache if necessary, only used /// by Group Top-N since it's distributed. fn update_vnode_bitmap(&mut self, _vnode_bitmap: Arc) { @@ -72,7 +70,7 @@ pub trait TopNExecutorBase: Send + 'static { /// The struct wraps a [`TopNExecutorBase`] pub struct TopNExecutorWrapper { - pub(super) input: BoxedExecutor, + pub(super) input: Executor, pub(super) ctx: ActorContextRef, pub(super) inner: E, } @@ -81,10 +79,6 @@ impl Execute for TopNExecutorWrapper where E: TopNExecutorBase, { - fn info(&self) -> &ExecutorInfo { - self.inner.info() - } - fn execute(self: Box) -> BoxedMessageStream { self.top_n_executor_execute().boxed() } diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index a2c51c92c6265..f8d905a91d815 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -17,40 +17,37 @@ use std::pin::Pin; use std::task::{Context, Poll}; use futures::stream::{FusedStream, FuturesUnordered}; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use futures_async_stream::try_stream; use pin_project::pin_project; use super::watermark::BufferedWatermarks; -use super::*; -use crate::executor::{BoxedMessageStream, ExecutorInfo}; +use super::{ + Barrier, BoxedMessageStream, Execute, Executor, Message, MessageStreamItem, StreamExecutorError, +}; /// `UnionExecutor` merges data from multiple inputs. pub struct UnionExecutor { - info: ExecutorInfo, - inputs: Vec, + inputs: Vec, } +// TODO() impl std::fmt::Debug for UnionExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("UnionExecutor") - .field("schema", &self.info.schema) - .field("pk_indices", &self.info.pk_indices) + // .field("schema", &self.info.schema) + // .field("pk_indices", &self.info.pk_indices) .finish() } } impl UnionExecutor { - pub fn new(info: ExecutorInfo, inputs: Vec) -> Self { - Self { info, inputs } + pub fn new(inputs: Vec) -> Self { + Self { inputs } } } impl Execute for UnionExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { let streams = self.inputs.into_iter().map(|e| e.execute()).collect(); merge(streams).boxed() @@ -157,8 +154,10 @@ mod tests { use async_stream::try_stream; use risingwave_common::array::stream_chunk::StreamChunkTestExt; use risingwave_common::array::StreamChunk; + use risingwave_common::types::{DataType, ScalarImpl}; use super::*; + use crate::executor::Watermark; #[tokio::test] async fn union() { diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs index 9ed9e3d8ae602..550d2a95970b8 100644 --- a/src/stream/src/executor/utils.rs +++ b/src/stream/src/executor/utils.rs @@ -20,21 +20,9 @@ use crate::executor::{BoxedMessageStream, Execute, ExecutorInfo}; use crate::task::{ActorId, FragmentId}; #[derive(Default)] -pub struct DummyExecutor { - pub info: ExecutorInfo, -} - -impl DummyExecutor { - pub fn new(info: ExecutorInfo) -> Self { - Self { info } - } -} +pub struct DummyExecutor; impl Execute for DummyExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { futures::stream::pending().boxed() } diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index c5e2097f8102a..52ed98490f1bc 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -18,15 +18,13 @@ use await_tree::InstrumentAwait; use futures::StreamExt; use futures_async_stream::try_stream; use risingwave_common::array::{DataChunk, Op, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::ensure; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; use tokio::sync::mpsc::UnboundedReceiver; -use super::{ - ActorContextRef, Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, - StreamExecutorError, -}; +use super::{ActorContextRef, Barrier, BoxedMessageStream, Execute, Message, StreamExecutorError}; use crate::task::CreateMviewProgress; const DEFAULT_CHUNK_SIZE: usize = 1024; @@ -35,8 +33,8 @@ const DEFAULT_CHUNK_SIZE: usize = 1024; /// May refractor with `BarrierRecvExecutor` in the near future. pub struct ValuesExecutor { ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, // Receiver of barrier channel. barrier_receiver: UnboundedReceiver, progress: CreateMviewProgress, @@ -48,14 +46,14 @@ impl ValuesExecutor { /// Currently hard-code the `pk_indices` as the last column. pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, progress: CreateMviewProgress, rows: Vec>, barrier_receiver: UnboundedReceiver, ) -> Self { Self { ctx, - info, + schema, progress, barrier_receiver, rows: rows.into_iter(), @@ -63,9 +61,9 @@ impl ValuesExecutor { } #[try_stream(ok = Message, error = StreamExecutorError)] - async fn into_stream(self) { + async fn execute_inner(self) { let Self { - info, + schema, mut progress, mut barrier_receiver, mut rows, @@ -95,7 +93,7 @@ impl ValuesExecutor { } } - let cardinality = info.schema.len(); + let cardinality = schema.len(); ensure!(cardinality > 0); while !rows.is_empty() { // We need a one row chunk rather than an empty chunk because constant @@ -104,7 +102,7 @@ impl ValuesExecutor { let one_row_chunk = DataChunk::new_dummy(1); let chunk_size = DEFAULT_CHUNK_SIZE.min(rows.len()); - let mut array_builders = info.schema.create_array_builders(chunk_size); + let mut array_builders = schema.create_array_builders(chunk_size); for row in rows.by_ref().take(chunk_size) { for (expr, builder) in row.into_iter().zip_eq_fast(&mut array_builders) { let out = expr.eval_infallible(&one_row_chunk).await; @@ -135,12 +133,8 @@ impl ValuesExecutor { } impl Execute for ValuesExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() + self.execute_inner().boxed() } } diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 3fbe43b2b19d2..96dcc512e62d0 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -36,7 +36,7 @@ use risingwave_storage::StateStore; use super::error::StreamExecutorError; use super::filter::FilterExecutor; -use super::{ActorContextRef, BoxedExecutor, Execute, ExecutorInfo, Message, StreamExecutorResult}; +use super::{ActorContextRef, Execute, Executor, ExecutorInfo, Message, StreamExecutorResult}; use crate::common::table::state_table::StateTable; use crate::executor::{expect_first_barrier, Watermark}; use crate::task::ActorEvalErrorReport; @@ -46,44 +46,46 @@ use crate::task::ActorEvalErrorReport; /// filtered. pub struct WatermarkFilterExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, /// The expression used to calculate the watermark value. watermark_expr: NonStrictExpression, /// The column we should generate watermark and filter on. event_time_col_idx: usize, table: StateTable, global_watermark_table: StorageTable, + + eval_error_report: ActorEvalErrorReport, } impl WatermarkFilterExecutor { pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + info: &ExecutorInfo, + input: Executor, watermark_expr: NonStrictExpression, event_time_col_idx: usize, table: StateTable, global_watermark_table: StorageTable, ) -> Self { + let eval_error_report = ActorEvalErrorReport { + actor_context: ctx.clone(), + identity: Arc::from(info.identity.as_ref()), + }; + Self { ctx, - info, input, watermark_expr, event_time_col_idx, table, global_watermark_table, + eval_error_report, } } } impl Execute for WatermarkFilterExecutor { - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } @@ -97,16 +99,11 @@ impl WatermarkFilterExecutor { event_time_col_idx, watermark_expr, ctx, - info, mut table, mut global_watermark_table, + eval_error_report, } = *self; - let eval_error_report = ActorEvalErrorReport { - actor_context: ctx.clone(), - identity: info.identity.into(), - }; - let watermark_type = watermark_expr.return_type(); assert_eq!( watermark_type, @@ -442,7 +439,7 @@ mod tests { async fn create_watermark_filter_executor( mem_state: MemoryStateStore, - ) -> (BoxedExecutor, MessageSender) { + ) -> (Box, MessageSender) { let schema = Schema { fields: vec![ Field::unnamed(DataType::Int16), // pk diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index e671f204f5a61..dddd94da5ab73 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -16,7 +16,7 @@ use std::sync::Arc; use futures::StreamExt; -use super::*; +use super::{ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, MessageStream}; mod epoch_check; mod epoch_provide; @@ -26,7 +26,7 @@ mod update_check; /// [`WrapperExecutor`] will do some sanity checks and logging for the wrapped executor. pub struct WrapperExecutor { - input: BoxedExecutor, + input: Executor, actor_ctx: ActorContextRef, @@ -35,7 +35,7 @@ pub struct WrapperExecutor { impl WrapperExecutor { pub fn new( - input: BoxedExecutor, + input: Executor, actor_ctx: ActorContextRef, enable_executor_row_count: bool, ) -> Self { @@ -88,10 +88,6 @@ impl WrapperExecutor { } impl Execute for WrapperExecutor { - fn info(&self) -> &ExecutorInfo { - self.input.info() - } - fn execute(self: Box) -> BoxedMessageStream { let info = Arc::new(self.input.info().clone()); Self::wrap( diff --git a/src/stream/src/from_proto/append_only_dedup.rs b/src/stream/src/from_proto/append_only_dedup.rs index 4ae2ee24ce798..020d13f3ff96b 100644 --- a/src/stream/src/from_proto/append_only_dedup.rs +++ b/src/stream/src/from_proto/append_only_dedup.rs @@ -20,7 +20,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{AppendOnlyDedupExecutor, BoxedExecutor}; +use crate::executor::{AppendOnlyDedupExecutor, Executor}; use crate::task::ExecutorParams; pub struct AppendOnlyDedupExecutorBuilder; @@ -32,18 +32,19 @@ impl ExecutorBuilder for AppendOnlyDedupExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let table = node.get_state_table()?; let vnodes = params.vnode_bitmap.map(Arc::new); let state_table = StateTable::from_table_catalog(table, store, vnodes).await; - Ok(Box::new(AppendOnlyDedupExecutor::new( + let exec = AppendOnlyDedupExecutor::new( + params.actor_context, input, + params.info.pk_indices.clone(), /* TODO(rc): should change to use `dedup_column_indices`, but need to check backward compatibility */ state_table, - params.info, - params.actor_context, params.watermark_epoch, params.executor_stats.clone(), - ))) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/barrier_recv.rs b/src/stream/src/from_proto/barrier_recv.rs index 032afde2ee1dc..b8c38f0f53d32 100644 --- a/src/stream/src/from_proto/barrier_recv.rs +++ b/src/stream/src/from_proto/barrier_recv.rs @@ -27,7 +27,7 @@ impl ExecutorBuilder for BarrierRecvExecutorBuilder { params: ExecutorParams, _node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { assert!( params.input.is_empty(), "barrier receiver should not have input" @@ -38,6 +38,7 @@ impl ExecutorBuilder for BarrierRecvExecutorBuilder { .local_barrier_manager .register_sender(params.actor_context.id, sender); - Ok(BarrierRecvExecutor::new(params.actor_context, params.info, barrier_receiver).boxed()) + let exec = BarrierRecvExecutor::new(params.actor_context, barrier_receiver); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/batch_query.rs b/src/stream/src/from_proto/batch_query.rs index 4fc492f7c984d..5e86d7e6d5b30 100644 --- a/src/stream/src/from_proto/batch_query.rs +++ b/src/stream/src/from_proto/batch_query.rs @@ -30,12 +30,12 @@ impl ExecutorBuilder for BatchQueryExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { if node.table_desc.is_none() { // used in sharing cdc source backfill as a dummy batch plan node let mut info = params.info; info.identity = "DummyBatchQueryExecutor".to_string(); - return Ok(Box::new(DummyExecutor::new(info))); + return Ok((info, DummyExecutor).into()); } let table_desc: &StorageTableDesc = node.get_table_desc()?; @@ -55,9 +55,11 @@ impl ExecutorBuilder for BatchQueryExecutorBuilder { ); assert_eq!(table.schema().data_types(), params.info.schema.data_types()); - let executor = - BatchQueryExecutor::new(table, params.env.config().developer.chunk_size, params.info); - - Ok(executor.boxed()) + let exec = BatchQueryExecutor::new( + table, + params.env.config().developer.chunk_size, + params.info.schema.clone(), + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/cdc_filter.rs b/src/stream/src/from_proto/cdc_filter.rs index 5747e1894d4a8..673f6a1c5e712 100644 --- a/src/stream/src/from_proto/cdc_filter.rs +++ b/src/stream/src/from_proto/cdc_filter.rs @@ -28,11 +28,12 @@ impl ExecutorBuilder for CdcFilterExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let search_condition = build_non_strict_from_prost(node.get_search_condition()?, params.eval_error_report)?; - Ok(FilterExecutor::new(params.actor_context, params.info, input, search_condition).boxed()) + let exec = FilterExecutor::new(params.actor_context, input, search_condition); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/dml.rs b/src/stream/src/from_proto/dml.rs index d7f21fda7bf2d..cc8836890ce7f 100644 --- a/src/stream/src/from_proto/dml.rs +++ b/src/stream/src/from_proto/dml.rs @@ -20,7 +20,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::error::StreamResult; use crate::executor::dml::DmlExecutor; -use crate::executor::BoxedExecutor; +use crate::executor::Executor; use crate::task::ExecutorParams; pub struct DmlExecutorBuilder; @@ -32,19 +32,19 @@ impl ExecutorBuilder for DmlExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); let table_id = TableId::new(node.table_id); let column_descs = node.column_descs.iter().map(Into::into).collect_vec(); - Ok(Box::new(DmlExecutor::new( - params.info, + let exec = DmlExecutor::new( upstream, params.env.dml_manager_ref(), table_id, node.table_version_id, column_descs, params.env.config().developer.chunk_size, - ))) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/dynamic_filter.rs b/src/stream/src/from_proto/dynamic_filter.rs index babb2680a9575..c09fd767ad9ef 100644 --- a/src/stream/src/from_proto/dynamic_filter.rs +++ b/src/stream/src/from_proto/dynamic_filter.rs @@ -32,7 +32,7 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [source_l, source_r]: [_; 2] = params.input.try_into().unwrap(); let key_l = node.get_left_key() as usize; @@ -62,7 +62,7 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { let left_table = node.get_left_table()?; let cleaned_by_watermark = left_table.get_cleaned_by_watermark(); - if cleaned_by_watermark { + let exec = if cleaned_by_watermark { let state_table_l = WatermarkCacheStateTable::from_table_catalog( node.get_left_table()?, store, @@ -70,9 +70,9 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { ) .await; - Ok(Box::new(DynamicFilterExecutor::new( + DynamicFilterExecutor::new( params.actor_context, - params.info, + ¶ms.info, source_l, source_r, key_l, @@ -83,14 +83,15 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { params.env.config().developer.chunk_size, condition_always_relax, cleaned_by_watermark, - ))) + ) + .boxed() } else { let state_table_l = StateTable::from_table_catalog(node.get_left_table()?, store, Some(vnodes)).await; - Ok(Box::new(DynamicFilterExecutor::new( + DynamicFilterExecutor::new( params.actor_context, - params.info, + ¶ms.info, source_l, source_r, key_l, @@ -101,7 +102,10 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { params.env.config().developer.chunk_size, condition_always_relax, cleaned_by_watermark, - ))) - } + ) + .boxed() + }; + + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/eowc_over_window.rs b/src/stream/src/from_proto/eowc_over_window.rs index 26879473dfbfc..81cd965cdef4c 100644 --- a/src/stream/src/from_proto/eowc_over_window.rs +++ b/src/stream/src/from_proto/eowc_over_window.rs @@ -21,7 +21,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Execute}; +use crate::executor::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Execute, Executor}; use crate::task::ExecutorParams; pub struct EowcOverWindowExecutorBuilder; @@ -33,7 +33,7 @@ impl ExecutorBuilder for EowcOverWindowExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let calls: Vec<_> = node .get_calls() @@ -54,18 +54,18 @@ impl ExecutorBuilder for EowcOverWindowExecutorBuilder { let state_table = StateTable::from_table_catalog_inconsistent_op(node.get_state_table()?, store, vnodes) .await; - Ok(EowcOverWindowExecutor::new(EowcOverWindowExecutorArgs { + let exec = EowcOverWindowExecutor::new(EowcOverWindowExecutorArgs { actor_ctx: params.actor_context, - info: params.info, input, + schema: params.info.schema.clone(), calls, partition_key_indices, order_key_index, state_table, watermark_epoch: params.watermark_epoch, - }) - .boxed()) + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/expand.rs b/src/stream/src/from_proto/expand.rs index 40bb972cdbd33..34d4313e24cb7 100644 --- a/src/stream/src/from_proto/expand.rs +++ b/src/stream/src/from_proto/expand.rs @@ -26,7 +26,7 @@ impl ExecutorBuilder for ExpandExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let column_subsets = node .column_subsets @@ -39,6 +39,6 @@ impl ExecutorBuilder for ExpandExecutorBuilder { .collect_vec() }) .collect_vec(); - Ok(ExpandExecutor::new(params.info, input, column_subsets).boxed()) + Ok((params.info, ExpandExecutor::new(input, column_subsets)).into()) } } diff --git a/src/stream/src/from_proto/filter.rs b/src/stream/src/from_proto/filter.rs index a82b9cc65dea7..a4efb953064b8 100644 --- a/src/stream/src/from_proto/filter.rs +++ b/src/stream/src/from_proto/filter.rs @@ -27,11 +27,12 @@ impl ExecutorBuilder for FilterExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let search_condition = build_non_strict_from_prost(node.get_search_condition()?, params.eval_error_report)?; - Ok(FilterExecutor::new(params.actor_context, params.info, input, search_condition).boxed()) + let exec = FilterExecutor::new(params.actor_context, input, search_condition); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/group_top_n.rs b/src/stream/src/from_proto/group_top_n.rs index a90e13d3f496b..51512cb9d94d3 100644 --- a/src/stream/src/from_proto/group_top_n.rs +++ b/src/stream/src/from_proto/group_top_n.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use risingwave_common::catalog::Schema; use risingwave_common::hash::{HashKey, HashKeyDispatcher}; use risingwave_common::types::DataType; use risingwave_common::util::sort_util::ColumnOrder; @@ -21,7 +22,7 @@ use risingwave_pb::stream_plan::GroupTopNNode; use super::*; use crate::common::table::state_table::StateTable; -use crate::executor::{ActorContextRef, AppendOnlyGroupTopNExecutor, GroupTopNExecutor}; +use crate::executor::{ActorContextRef, AppendOnlyGroupTopNExecutor, Executor, GroupTopNExecutor}; use crate::task::AtomicU64Ref; pub struct GroupTopNExecutorBuilder; @@ -33,7 +34,7 @@ impl ExecutorBuilder for GroupTopNExecutorBuilder StreamResult { + ) -> StreamResult { let group_by: Vec = node .get_group_key() .iter() @@ -61,7 +62,7 @@ impl ExecutorBuilder for GroupTopNExecutorBuilder ExecutorBuilder for GroupTopNExecutorBuilder { - input: BoxedExecutor, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -94,7 +95,7 @@ struct GroupTopNExecutorDispatcherArgs { } impl HashKeyDispatcher for GroupTopNExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { macro_rules! build { @@ -102,7 +103,7 @@ impl HashKeyDispatcher for GroupTopNExecutorDispatcherArgs { Ok($excutor::::new( self.input, self.ctx, - self.info, + self.schema, self.storage_key, self.offset_and_limit, self.order_by, diff --git a/src/stream/src/from_proto/hash_agg.rs b/src/stream/src/from_proto/hash_agg.rs index 52390b6806d0e..3331640519617 100644 --- a/src/stream/src/from_proto/hash_agg.rs +++ b/src/stream/src/from_proto/hash_agg.rs @@ -35,7 +35,7 @@ pub struct HashAggExecutorDispatcherArgs { } impl HashKeyDispatcher for HashAggExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { Ok(HashAggExecutor::::new(self.args)?.boxed()) @@ -55,7 +55,7 @@ impl ExecutorBuilder for HashAggExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let group_key_indices = node .get_group_key() .iter() @@ -93,13 +93,13 @@ impl ExecutorBuilder for HashAggExecutorBuilder { build_distinct_dedup_table_from_proto(node.get_distinct_dedup_tables(), store, vnodes) .await; - HashAggExecutorDispatcherArgs { + let exec = HashAggExecutorDispatcherArgs { args: AggExecutorArgs { version: node.version(), input, actor_ctx: params.actor_context, - info: params.info, + info: params.info.clone(), extreme_cache_size: params.env.config().developer.unsafe_extreme_cache_size, @@ -122,6 +122,7 @@ impl ExecutorBuilder for HashAggExecutorBuilder { }, group_key_types, } - .dispatch() + .dispatch()?; + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/hash_join.rs b/src/stream/src/from_proto/hash_join.rs index adfd667e920f2..ad69fd1ecd5ac 100644 --- a/src/stream/src/from_proto/hash_join.rs +++ b/src/stream/src/from_proto/hash_join.rs @@ -27,7 +27,7 @@ use super::*; use crate::common::table::state_table::StateTable; use crate::executor::hash_join::*; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{ActorContextRef, JoinType}; +use crate::executor::{ActorContextRef, Executor, JoinType}; use crate::task::AtomicU64Ref; pub struct HashJoinExecutorBuilder; @@ -39,7 +39,7 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let is_append_only = node.is_append_only; let vnodes = Arc::new(params.vnode_bitmap.expect("vnodes not set for hash join")); @@ -136,7 +136,7 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { let args = HashJoinExecutorDispatcherArgs { ctx: params.actor_context, - info: params.info, + info: params.info.clone(), source_l, source_r, params_l, @@ -157,15 +157,16 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { chunk_size: params.env.config().developer.chunk_size, }; - args.dispatch() + let exec = args.dispatch()?; + Ok((params.info, exec).into()) } } struct HashJoinExecutorDispatcherArgs { ctx: ActorContextRef, info: ExecutorInfo, - source_l: BoxedExecutor, - source_r: BoxedExecutor, + source_l: Executor, + source_r: Executor, params_l: JoinParams, params_r: JoinParams, null_safe: Vec, @@ -185,34 +186,33 @@ struct HashJoinExecutorDispatcherArgs { } impl HashKeyDispatcher for HashJoinExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { /// This macro helps to fill the const generic type parameter. macro_rules! build { ($join_type:ident) => { - Ok(Box::new( - HashJoinExecutor::::new( - self.ctx, - self.info, - self.source_l, - self.source_r, - self.params_l, - self.params_r, - self.null_safe, - self.output_indices, - self.cond, - self.inequality_pairs, - self.state_table_l, - self.degree_state_table_l, - self.state_table_r, - self.degree_state_table_r, - self.lru_manager, - self.is_append_only, - self.metrics, - self.chunk_size, - ), - )) + Ok(HashJoinExecutor::::new( + self.ctx, + self.info, + self.source_l, + self.source_r, + self.params_l, + self.params_r, + self.null_safe, + self.output_indices, + self.cond, + self.inequality_pairs, + self.state_table_l, + self.degree_state_table_l, + self.state_table_r, + self.degree_state_table_r, + self.lru_manager, + self.is_append_only, + self.metrics, + self.chunk_size, + ) + .boxed()) }; } match self.join_type_proto { diff --git a/src/stream/src/from_proto/hop_window.rs b/src/stream/src/from_proto/hop_window.rs index a56cbc2d8c976..2598ae927608e 100644 --- a/src/stream/src/from_proto/hop_window.rs +++ b/src/stream/src/from_proto/hop_window.rs @@ -27,16 +27,8 @@ impl ExecutorBuilder for HopWindowExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { - let ExecutorParams { - actor_context, - info, - input, - env, - .. - } = params; - - let input = input.into_iter().next().unwrap(); + ) -> StreamResult { + let input = params.input.into_iter().next().unwrap(); // TODO: reuse the schema derivation with frontend. let output_indices = node .get_output_indices() @@ -59,11 +51,10 @@ impl ExecutorBuilder for HopWindowExecutorBuilder { let window_slide = node.get_window_slide()?.into(); let window_size = node.get_window_size()?.into(); - let chunk_size = env.config().developer.chunk_size; + let chunk_size = params.env.config().developer.chunk_size; - Ok(HopWindowExecutor::new( - actor_context, - info, + let exec = HopWindowExecutor::new( + params.actor_context, input, time_col, window_slide, @@ -72,7 +63,7 @@ impl ExecutorBuilder for HopWindowExecutorBuilder { window_end_exprs, output_indices, chunk_size, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/lookup.rs b/src/stream/src/from_proto/lookup.rs index 1f873f31127c5..dc7f7e3c49dfe 100644 --- a/src/stream/src/from_proto/lookup.rs +++ b/src/stream/src/from_proto/lookup.rs @@ -30,7 +30,7 @@ impl ExecutorBuilder for LookupExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let lookup = node; let [stream, arrangement]: [_; 2] = params.input.try_into().unwrap(); @@ -69,9 +69,9 @@ impl ExecutorBuilder for LookupExecutorBuilder { table_desc, ); - Ok(Box::new(LookupExecutor::new(LookupExecutorParams { + let exec = LookupExecutor::new(LookupExecutorParams { ctx: params.actor_context, - info: params.info, + info: params.info.clone(), arrangement, stream, arrangement_col_descs, @@ -83,6 +83,7 @@ impl ExecutorBuilder for LookupExecutorBuilder { storage_table, watermark_epoch: params.watermark_epoch, chunk_size: params.env.config().developer.chunk_size, - }))) + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/lookup_union.rs b/src/stream/src/from_proto/lookup_union.rs index e9cc0ed33311a..93e847609cae2 100644 --- a/src/stream/src/from_proto/lookup_union.rs +++ b/src/stream/src/from_proto/lookup_union.rs @@ -26,7 +26,8 @@ impl ExecutorBuilder for LookupUnionExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { - Ok(LookupUnionExecutor::new(params.info, params.input, node.order.clone()).boxed()) + ) -> StreamResult { + let exec = LookupUnionExecutor::new(params.input, node.order.clone()); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/merge.rs b/src/stream/src/from_proto/merge.rs index aa0a6a9d2bb92..eded1f59d294e 100644 --- a/src/stream/src/from_proto/merge.rs +++ b/src/stream/src/from_proto/merge.rs @@ -27,7 +27,7 @@ impl ExecutorBuilder for MergeExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let upstreams = node.get_upstream_actor_id(); let upstream_fragment_id = node.get_upstream_fragment_id(); @@ -56,10 +56,9 @@ impl ExecutorBuilder for MergeExecutorBuilder { DispatcherType::NoShuffle => true, }; - if always_single_input { - Ok(ReceiverExecutor::new( + let exec = if always_single_input { + ReceiverExecutor::new( params.actor_context, - params.info, params.fragment_id, upstream_fragment_id, inputs.into_iter().exactly_one().unwrap(), @@ -67,11 +66,10 @@ impl ExecutorBuilder for MergeExecutorBuilder { params.operator_id, params.executor_stats.clone(), ) - .boxed()) + .boxed() } else { - Ok(MergeExecutor::new( + MergeExecutor::new( params.actor_context, - params.info, params.fragment_id, upstream_fragment_id, inputs, @@ -79,7 +77,8 @@ impl ExecutorBuilder for MergeExecutorBuilder { params.operator_id, params.executor_stats.clone(), ) - .boxed()) - } + .boxed() + }; + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/mod.rs b/src/stream/src/from_proto/mod.rs index 9d87603ce03e7..d82d27299a32a 100644 --- a/src/stream/src/from_proto/mod.rs +++ b/src/stream/src/from_proto/mod.rs @@ -93,7 +93,7 @@ use self::top_n::*; use self::union::*; use self::watermark_filter::WatermarkFilterBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Execute, ExecutorInfo}; +use crate::executor::{Execute, Executor, ExecutorInfo}; use crate::from_proto::subscription::SubscriptionExecutorBuilder; use crate::from_proto::values::ValuesExecutorBuilder; use crate::task::ExecutorParams; @@ -101,12 +101,14 @@ use crate::task::ExecutorParams; trait ExecutorBuilder { type Node; - /// Create a [`BoxedExecutor`] from [`StreamNode`]. + // TODO(): rename + // TODO(): async trait + /// Create a boxed [`Execute`] from [`StreamNode`]. fn new_boxed_executor( params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> impl std::future::Future> + Send; + ) -> impl std::future::Future> + Send; } macro_rules! build_executor { @@ -127,7 +129,7 @@ pub async fn create_executor( params: ExecutorParams, node: &StreamNode, store: impl StateStore, -) -> StreamResult { +) -> StreamResult { build_executor! { params, node, diff --git a/src/stream/src/from_proto/mview.rs b/src/stream/src/from_proto/mview.rs index ecc54b20d8535..80aab576435ca 100644 --- a/src/stream/src/from_proto/mview.rs +++ b/src/stream/src/from_proto/mview.rs @@ -32,7 +32,7 @@ impl ExecutorBuilder for MaterializeExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let order_key = node @@ -51,7 +51,7 @@ impl ExecutorBuilder for MaterializeExecutorBuilder { ($SD:ident) => { MaterializeExecutor::<_, $SD>::new( input, - params.info, + params.info.schema.clone(), store, order_key, params.actor_context, @@ -66,13 +66,13 @@ impl ExecutorBuilder for MaterializeExecutorBuilder { }; } - let executor = if versioned { + let exec = if versioned { new_executor!(ColumnAwareSerde) } else { new_executor!(BasicSerde) }; - Ok(executor) + Ok((params.info, exec).into()) } } @@ -85,7 +85,7 @@ impl ExecutorBuilder for ArrangeExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let keys = node @@ -102,9 +102,9 @@ impl ExecutorBuilder for ArrangeExecutorBuilder { let vnodes = params.vnode_bitmap.map(Arc::new); let conflict_behavior = ConflictBehavior::from_protobuf(&table.handle_pk_conflict_behavior()); - let executor = MaterializeExecutor::<_, BasicSerde>::new( + let exec = MaterializeExecutor::<_, BasicSerde>::new( input, - params.info, + params.info.schema.clone(), store, keys, params.actor_context, @@ -116,6 +116,6 @@ impl ExecutorBuilder for ArrangeExecutorBuilder { ) .await; - Ok(executor.boxed()) + Ok((params.info, exec.boxed()).into()) } } diff --git a/src/stream/src/from_proto/no_op.rs b/src/stream/src/from_proto/no_op.rs index 22dae0020abf5..31494d771b903 100644 --- a/src/stream/src/from_proto/no_op.rs +++ b/src/stream/src/from_proto/no_op.rs @@ -17,7 +17,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Execute, NoOpExecutor}; +use crate::executor::{Executor, NoOpExecutor}; use crate::task::ExecutorParams; pub struct NoOpExecutorBuilder; @@ -29,8 +29,8 @@ impl ExecutorBuilder for NoOpExecutorBuilder { params: ExecutorParams, _node: &NoOpNode, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); - Ok(NoOpExecutor::new(params.actor_context, params.info, input).boxed()) + Ok((params.info, NoOpExecutor::new(params.actor_context, input)).into()) } } diff --git a/src/stream/src/from_proto/now.rs b/src/stream/src/from_proto/now.rs index f622cdef1343e..ccfff43f3cf49 100644 --- a/src/stream/src/from_proto/now.rs +++ b/src/stream/src/from_proto/now.rs @@ -19,7 +19,7 @@ use tokio::sync::mpsc::unbounded_channel; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, NowExecutor}; +use crate::executor::{Execute, Executor, NowExecutor}; use crate::task::ExecutorParams; pub struct NowExecutorBuilder; @@ -31,7 +31,7 @@ impl ExecutorBuilder for NowExecutorBuilder { params: ExecutorParams, node: &NowNode, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let (sender, barrier_receiver) = unbounded_channel(); params .local_barrier_manager @@ -40,10 +40,11 @@ impl ExecutorBuilder for NowExecutorBuilder { let state_table = StateTable::from_table_catalog(node.get_state_table()?, store, None).await; - Ok(Box::new(NowExecutor::new( - params.info, + let exec = NowExecutor::new( + params.info.schema.data_types(), barrier_receiver, state_table, - ))) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/over_window.rs b/src/stream/src/from_proto/over_window.rs index ddc5db3cc2582..b75d2d66fd646 100644 --- a/src/stream/src/from_proto/over_window.rs +++ b/src/stream/src/from_proto/over_window.rs @@ -23,7 +23,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Execute, OverWindowExecutor, OverWindowExecutorArgs}; +use crate::executor::{Execute, Executor, OverWindowExecutor, OverWindowExecutorArgs}; use crate::task::ExecutorParams; pub struct OverWindowExecutorBuilder; @@ -35,7 +35,7 @@ impl ExecutorBuilder for OverWindowExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let calls: Vec<_> = node .get_calls() @@ -60,12 +60,12 @@ impl ExecutorBuilder for OverWindowExecutorBuilder { )); let state_table = StateTable::from_table_catalog(node.get_state_table()?, store, vnodes).await; - Ok(OverWindowExecutor::new(OverWindowExecutorArgs { + let exec = OverWindowExecutor::new(OverWindowExecutorArgs { actor_ctx: params.actor_context, - info: params.info, input, + schema: params.info.schema.clone(), calls, partition_key_indices, order_key_indices, @@ -79,7 +79,7 @@ impl ExecutorBuilder for OverWindowExecutorBuilder { cache_policy: OverWindowCachePolicy::from_protobuf( node.get_cache_policy().unwrap_or_default(), ), - }) - .boxed()) + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/project.rs b/src/stream/src/from_proto/project.rs index 6fc9ada39aed8..d7f96c4dffcbf 100644 --- a/src/stream/src/from_proto/project.rs +++ b/src/stream/src/from_proto/project.rs @@ -30,7 +30,7 @@ impl ExecutorBuilder for ProjectExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let project_exprs: Vec<_> = node .get_select_list() @@ -60,15 +60,14 @@ impl ExecutorBuilder for ProjectExecutorBuilder { ) }); let materialize_selectivity_threshold = if extremely_light { 0.0 } else { 0.5 }; - Ok(ProjectExecutor::new( + let exec = ProjectExecutor::new( params.actor_context, - params.info, input, project_exprs, watermark_derivations, nondecreasing_expr_indices, materialize_selectivity_threshold, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/project_set.rs b/src/stream/src/from_proto/project_set.rs index b98e9e467ccf2..c2338394b33ef 100644 --- a/src/stream/src/from_proto/project_set.rs +++ b/src/stream/src/from_proto/project_set.rs @@ -29,7 +29,7 @@ impl ExecutorBuilder for ProjectSetExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let select_list: Vec<_> = node .get_select_list() @@ -55,15 +55,14 @@ impl ExecutorBuilder for ProjectSetExecutorBuilder { .collect(); let chunk_size = params.env.config().developer.chunk_size; - Ok(ProjectSetExecutor::new( + let exec = ProjectSetExecutor::new( params.actor_context, - params.info, input, select_list, chunk_size, watermark_derivations, nondecreasing_expr_indices, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/row_id_gen.rs b/src/stream/src/from_proto/row_id_gen.rs index 1333a99aebfa5..ab87566547ac6 100644 --- a/src/stream/src/from_proto/row_id_gen.rs +++ b/src/stream/src/from_proto/row_id_gen.rs @@ -18,7 +18,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::error::StreamResult; use crate::executor::row_id_gen::RowIdGenExecutor; -use crate::executor::BoxedExecutor; +use crate::executor::Executor; use crate::task::ExecutorParams; pub struct RowIdGenExecutorBuilder; @@ -30,19 +30,18 @@ impl ExecutorBuilder for RowIdGenExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); tracing::debug!("row id gen executor: {:?}", params.vnode_bitmap); let vnodes = params .vnode_bitmap .expect("vnodes not set for row id gen executor"); - let executor = RowIdGenExecutor::new( + let exec = RowIdGenExecutor::new( params.actor_context, - params.info, upstream, node.row_id_index as _, vnodes, ); - Ok(Box::new(executor)) + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/simple_agg.rs b/src/stream/src/from_proto/simple_agg.rs index 209a08daf6a22..16809edb8bcaf 100644 --- a/src/stream/src/from_proto/simple_agg.rs +++ b/src/stream/src/from_proto/simple_agg.rs @@ -34,7 +34,7 @@ impl ExecutorBuilder for SimpleAggExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let agg_calls: Vec = node .get_agg_calls() @@ -55,12 +55,12 @@ impl ExecutorBuilder for SimpleAggExecutorBuilder { build_distinct_dedup_table_from_proto(node.get_distinct_dedup_tables(), store, None) .await; - Ok(SimpleAggExecutor::new(AggExecutorArgs { + let exec = SimpleAggExecutor::new(AggExecutorArgs { version: node.version(), input, actor_ctx: params.actor_context, - info: params.info, + info: params.info.clone(), extreme_cache_size: params.env.config().developer.unsafe_extreme_cache_size, @@ -71,7 +71,8 @@ impl ExecutorBuilder for SimpleAggExecutorBuilder { distinct_dedup_tables, watermark_epoch: params.watermark_epoch, extra: SimpleAggExecutorExtraArgs {}, - })? - .boxed()) + })?; + + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/sink.rs b/src/stream/src/from_proto/sink.rs index 87d63cd80f599..884925d4a35ab 100644 --- a/src/stream/src/from_proto/sink.rs +++ b/src/stream/src/from_proto/sink.rs @@ -103,7 +103,7 @@ impl ExecutorBuilder for SinkExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input_executor]: [_; 1] = params.input.try_into().unwrap(); let sink_desc = node.sink_desc.as_ref().unwrap(); @@ -189,23 +189,22 @@ impl ExecutorBuilder for SinkExecutorBuilder { connector, sink_id.sink_id, params.executor_id ); - match node.log_store_type() { + let exec = match node.log_store_type() { // Default value is the normal in memory log store to be backward compatible with the // previously unset value SinkLogStoreType::InMemoryLogStore | SinkLogStoreType::Unspecified => { let factory = BoundedInMemLogStoreFactory::new(1); - Ok(Box::new( - SinkExecutor::new( - params.actor_context, - params.info, - input_executor, - sink_write_param, - sink_param, - columns, - factory, - ) - .await?, - )) + SinkExecutor::new( + params.actor_context, + params.info.clone(), + input_executor, + sink_write_param, + sink_param, + columns, + factory, + ) + .await? + .boxed() } SinkLogStoreType::KvLogStore => { let metrics = KvLogStoreMetrics::new( @@ -230,19 +229,20 @@ impl ExecutorBuilder for SinkExecutorBuilder { pk_info, ); - Ok(Box::new( - SinkExecutor::new( - params.actor_context, - params.info, - input_executor, - sink_write_param, - sink_param, - columns, - factory, - ) - .await?, - )) + SinkExecutor::new( + params.actor_context, + params.info.clone(), + input_executor, + sink_write_param, + sink_param, + columns, + factory, + ) + .await? + .boxed() } - } + }; + + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/sort.rs b/src/stream/src/from_proto/sort.rs index dcb59f77f9549..e901d66e5d8ff 100644 --- a/src/stream/src/from_proto/sort.rs +++ b/src/stream/src/from_proto/sort.rs @@ -29,18 +29,19 @@ impl ExecutorBuilder for SortExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let vnodes = Arc::new(params.vnode_bitmap.expect("vnodes not set for sort")); let state_table = StateTable::from_table_catalog(node.get_state_table()?, store, Some(vnodes)).await; - Ok(Box::new(SortExecutor::new(SortExecutorArgs { + let exec = SortExecutor::new(SortExecutorArgs { actor_ctx: params.actor_context, - info: params.info, + schema: params.info.schema.clone(), input, buffer_table: state_table, chunk_size: params.env.config().developer.chunk_size, sort_column_index: node.sort_column_index as _, - }))) + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/source/fs_fetch.rs b/src/stream/src/from_proto/source/fs_fetch.rs index fa2f4475b3759..2a69be490c137 100644 --- a/src/stream/src/from_proto/source/fs_fetch.rs +++ b/src/stream/src/from_proto/source/fs_fetch.rs @@ -25,7 +25,7 @@ use risingwave_storage::StateStore; use crate::error::StreamResult; use crate::executor::{ - BoxedExecutor, Execute, FlowControlExecutor, FsFetchExecutor, SourceStateTableHandler, + Execute, Executor, FlowControlExecutor, FsFetchExecutor, SourceStateTableHandler, StreamSourceCore, }; use crate::from_proto::ExecutorBuilder; @@ -40,7 +40,7 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); let source = node.node_inner.as_ref().unwrap(); @@ -88,11 +88,10 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { state_table_handler, ); - let executor = match properties { + let exec = match properties { risingwave_connector::source::ConnectorProperties::Gcs(_) => { FsFetchExecutor::<_, OpendalGcs>::new( params.actor_context.clone(), - params.info, stream_source_core, upstream, source_ctrl_opts, @@ -103,7 +102,6 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { risingwave_connector::source::ConnectorProperties::OpendalS3(_) => { FsFetchExecutor::<_, OpendalS3>::new( params.actor_context.clone(), - params.info, stream_source_core, upstream, source_ctrl_opts, @@ -114,7 +112,6 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { risingwave_connector::source::ConnectorProperties::PosixFs(_) => { FsFetchExecutor::<_, OpendalPosixFs>::new( params.actor_context.clone(), - params.info, stream_source_core, upstream, source_ctrl_opts, @@ -124,7 +121,11 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { } _ => unreachable!(), }; + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); + let rate_limit = source.rate_limit.map(|x| x as _); - Ok(FlowControlExecutor::new(executor, params.actor_context, rate_limit).boxed()) + let exec = FlowControlExecutor::new((info, exec).into(), params.actor_context, rate_limit); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/source/trad_source.rs b/src/stream/src/from_proto/source/trad_source.rs index 28d923ffb69cc..5a5d49907c6f4 100644 --- a/src/stream/src/from_proto/source/trad_source.rs +++ b/src/stream/src/from_proto/source/trad_source.rs @@ -45,7 +45,7 @@ impl ExecutorBuilder for SourceExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let (sender, barrier_receiver) = unbounded_channel(); params .local_barrier_manager @@ -53,7 +53,7 @@ impl ExecutorBuilder for SourceExecutorBuilder { let system_params = params.env.system_params_manager_ref().get_params(); if let Some(source) = &node.source_inner { - let executor = { + let exec = { let source_id = TableId::new(source.source_id); let source_name = source.source_name.clone(); let mut source_info = source.get_info()?.clone(); @@ -191,7 +191,6 @@ impl ExecutorBuilder for SourceExecutorBuilder { #[expect(deprecated)] crate::executor::FsSourceExecutor::new( params.actor_context.clone(), - params.info, stream_source_core, params.executor_stats, barrier_receiver, @@ -202,7 +201,6 @@ impl ExecutorBuilder for SourceExecutorBuilder { } else if is_fs_v2_connector { FsListExecutor::new( params.actor_context.clone(), - params.info.clone(), Some(stream_source_core), params.executor_stats.clone(), barrier_receiver, @@ -214,7 +212,6 @@ impl ExecutorBuilder for SourceExecutorBuilder { } else { SourceExecutor::new( params.actor_context.clone(), - params.info.clone(), Some(stream_source_core), params.executor_stats.clone(), barrier_receiver, @@ -225,14 +222,18 @@ impl ExecutorBuilder for SourceExecutorBuilder { .boxed() } }; + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); + let rate_limit = source.rate_limit.map(|x| x as _); - Ok(FlowControlExecutor::new(executor, params.actor_context, rate_limit).boxed()) + let exec = + FlowControlExecutor::new((info, exec).into(), params.actor_context, rate_limit); + Ok((params.info, exec).into()) } else { // If there is no external stream source, then no data should be persisted. We pass a // `PanicStateStore` type here for indication. - Ok(SourceExecutor::::new( + let exec = SourceExecutor::::new( params.actor_context, - params.info, None, params.executor_stats, barrier_receiver, @@ -240,8 +241,8 @@ impl ExecutorBuilder for SourceExecutorBuilder { // we don't expect any data in, so no need to set chunk_sizes SourceCtrlOpts::default(), params.env.connector_params(), - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } } diff --git a/src/stream/src/from_proto/stateless_simple_agg.rs b/src/stream/src/from_proto/stateless_simple_agg.rs index 87617b42f83e4..2ce5e612737e8 100644 --- a/src/stream/src/from_proto/stateless_simple_agg.rs +++ b/src/stream/src/from_proto/stateless_simple_agg.rs @@ -27,7 +27,7 @@ impl ExecutorBuilder for StatelessSimpleAggExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let agg_calls: Vec = node .get_agg_calls() @@ -35,9 +35,12 @@ impl ExecutorBuilder for StatelessSimpleAggExecutorBuilder { .map(AggCall::from_protobuf) .try_collect()?; - Ok( - StatelessSimpleAggExecutor::new(params.actor_context, params.info, input, agg_calls)? - .boxed(), - ) + let exec = StatelessSimpleAggExecutor::new( + params.actor_context, + input, + params.info.schema.clone(), + agg_calls, + )?; + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/stream_cdc_scan.rs b/src/stream/src/from_proto/stream_cdc_scan.rs index 66e08c2bb4695..2736fdd712cb7 100644 --- a/src/stream/src/from_proto/stream_cdc_scan.rs +++ b/src/stream/src/from_proto/stream_cdc_scan.rs @@ -23,7 +23,7 @@ use risingwave_pb::stream_plan::StreamCdcScanNode; use super::*; use crate::common::table::state_table::StateTable; -use crate::executor::{CdcBackfillExecutor, ExternalStorageTable, FlowControlExecutor}; +use crate::executor::{CdcBackfillExecutor, Executor, ExternalStorageTable, FlowControlExecutor}; pub struct StreamCdcScanExecutorBuilder; @@ -34,7 +34,7 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); let output_indices = node @@ -95,9 +95,8 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder { .map(|x| std::cmp::min(x as usize, chunk_size)) .unwrap_or(chunk_size); - let executor = CdcBackfillExecutor::new( + let exec = CdcBackfillExecutor::new( params.actor_context.clone(), - params.info, external_table, upstream, output_indices, @@ -106,14 +105,15 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder { state_table, backfill_chunk_size, disable_backfill, - ) - .boxed(); + ); + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); - Ok(FlowControlExecutor::new( - executor, + let exec = FlowControlExecutor::new( + (info, exec).into(), params.actor_context, node.rate_limit.map(|x| x as _), - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/stream_scan.rs b/src/stream/src/from_proto/stream_scan.rs index c3f15802afc8d..5f0ae484e04c8 100644 --- a/src/stream/src/from_proto/stream_scan.rs +++ b/src/stream/src/from_proto/stream_scan.rs @@ -37,7 +37,7 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [upstream, snapshot]: [_; 2] = params.input.try_into().unwrap(); // For reporting the progress. let progress = params @@ -50,13 +50,13 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { .map(|&i| i as usize) .collect_vec(); - let executor = match node.stream_scan_type() { + let exec = match node.stream_scan_type() { StreamScanType::Chain | StreamScanType::UpstreamOnly => { let upstream_only = matches!(node.stream_scan_type(), StreamScanType::UpstreamOnly); - ChainExecutor::new(params.info, snapshot, upstream, progress, upstream_only).boxed() + ChainExecutor::new(snapshot, upstream, progress, upstream_only).boxed() } StreamScanType::Rearrange => { - RearrangedChainExecutor::new(params.info, snapshot, upstream, progress).boxed() + RearrangedChainExecutor::new(snapshot, upstream, progress).boxed() } StreamScanType::Backfill => { @@ -83,7 +83,6 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { StorageTable::new_partial(state_store.clone(), column_ids, vnodes, table_desc); BackfillExecutor::new( - params.info, upstream_table, upstream, state_table, @@ -126,7 +125,6 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { ) .await; ArrangementBackfillExecutor::<_, $SD>::new( - params.info, upstream_table, upstream, state_table, @@ -147,11 +145,14 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { } StreamScanType::Unspecified => unreachable!(), }; - Ok(FlowControlExecutor::new( - executor, + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); + + let exec = FlowControlExecutor::new( + (info, exec).into(), params.actor_context, node.rate_limit.map(|x| x as _), - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/subscription.rs b/src/stream/src/from_proto/subscription.rs index 4d5e6eb6855b1..ff0c1f8c1d084 100644 --- a/src/stream/src/from_proto/subscription.rs +++ b/src/stream/src/from_proto/subscription.rs @@ -21,7 +21,7 @@ use crate::common::log_store_impl::kv_log_store::serde::LogStoreRowSerde; use crate::common::log_store_impl::kv_log_store::KV_LOG_STORE_V2_INFO; use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, SubscriptionExecutor}; +use crate::executor::{Executor, SubscriptionExecutor}; pub struct SubscriptionExecutorBuilder; @@ -32,7 +32,7 @@ impl ExecutorBuilder for SubscriptionExecutorBuilder { params: crate::task::ExecutorParams, node: &Self::Node, state_store: impl risingwave_storage::StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let table_id = TableId::new(node.log_store_table.as_ref().unwrap().id); let local_state_store = state_store @@ -61,15 +61,13 @@ impl ExecutorBuilder for SubscriptionExecutorBuilder { let log_store_identity = format!("subscription-executor[{}]", params.executor_id); let log_store = SubscriptionLogStoreWriter::new(table_id, local_state_store, serde, log_store_identity); - Ok(Box::new( - SubscriptionExecutor::new( - params.actor_context, - params.info, - input, - log_store, - node.retention_seconds, - ) - .await?, - )) + let exec = SubscriptionExecutor::new( + params.actor_context, + input, + log_store, + node.retention_seconds, + ) + .await?; + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/temporal_join.rs b/src/stream/src/from_proto/temporal_join.rs index 7c91b8c1d261d..15badec97e5cc 100644 --- a/src/stream/src/from_proto/temporal_join.rs +++ b/src/stream/src/from_proto/temporal_join.rs @@ -35,7 +35,7 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let table_desc: &StorageTableDesc = node.get_table_desc()?; let table = { let column_ids = table_desc @@ -101,7 +101,7 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder { let dispatcher_args = TemporalJoinExecutorDispatcherArgs { ctx: params.actor_context, - info: params.info, + info: params.info.clone(), left: source_l, right: source_r, right_table: table, @@ -119,15 +119,15 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder { join_key_data_types, }; - dispatcher_args.dispatch() + Ok((params.info, dispatcher_args.dispatch()?).into()) } } struct TemporalJoinExecutorDispatcherArgs { ctx: ActorContextRef, info: ExecutorInfo, - left: BoxedExecutor, - right: BoxedExecutor, + left: Executor, + right: Executor, right_table: StorageTable, left_join_keys: Vec, right_join_keys: Vec, @@ -144,7 +144,7 @@ struct TemporalJoinExecutorDispatcherArgs { } impl HashKeyDispatcher for TemporalJoinExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { /// This macro helps to fill the const generic type parameter. diff --git a/src/stream/src/from_proto/top_n.rs b/src/stream/src/from_proto/top_n.rs index 5c74f9f97e0eb..10071534e57df 100644 --- a/src/stream/src/from_proto/top_n.rs +++ b/src/stream/src/from_proto/top_n.rs @@ -30,7 +30,7 @@ impl ExecutorBuilder for TopNExecutorBuilder StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let table = node.get_table()?; @@ -52,7 +52,7 @@ impl ExecutorBuilder for TopNExecutorBuilder::new( input, params.actor_context, - params.info, + params.info.schema.clone(), storage_key, (node.offset as usize, node.limit as usize), order_by, @@ -62,11 +62,12 @@ impl ExecutorBuilder for TopNExecutorBuilder> = match (APPEND_ONLY, node.with_ties) { (true, true) => build!(AppendOnlyTopNExecutor, true), (true, false) => build!(AppendOnlyTopNExecutor, false), (false, true) => build!(TopNExecutor, true), (false, false) => build!(TopNExecutor, false), - } + }; + Ok((params.info, exec?).into()) } } diff --git a/src/stream/src/from_proto/union.rs b/src/stream/src/from_proto/union.rs index e11ea8f3f2d21..1360f92c1c83b 100644 --- a/src/stream/src/from_proto/union.rs +++ b/src/stream/src/from_proto/union.rs @@ -26,7 +26,7 @@ impl ExecutorBuilder for UnionExecutorBuilder { params: ExecutorParams, _node: &Self::Node, _store: impl StateStore, - ) -> StreamResult { - Ok(UnionExecutor::new(params.info, params.input).boxed()) + ) -> StreamResult { + Ok((params.info, UnionExecutor::new(params.input)).into()) } } diff --git a/src/stream/src/from_proto/values.rs b/src/stream/src/from_proto/values.rs index b3ed75e4f0903..28e9ca086414b 100644 --- a/src/stream/src/from_proto/values.rs +++ b/src/stream/src/from_proto/values.rs @@ -20,7 +20,7 @@ use tokio::sync::mpsc::unbounded_channel; use super::ExecutorBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, ValuesExecutor}; +use crate::executor::{Executor, ValuesExecutor}; use crate::task::ExecutorParams; /// Build a `ValuesExecutor` for stream. As is a leaf, current workaround registers a `sender` for @@ -34,7 +34,7 @@ impl ExecutorBuilder for ValuesExecutorBuilder { params: ExecutorParams, node: &ValuesNode, _store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let (sender, barrier_receiver) = unbounded_channel(); params .local_barrier_manager @@ -55,12 +55,13 @@ impl ExecutorBuilder for ValuesExecutorBuilder { .collect_vec() }) .collect_vec(); - Ok(Box::new(ValuesExecutor::new( + let exec = ValuesExecutor::new( params.actor_context, - params.info, + params.info.schema.clone(), progress, rows, barrier_receiver, - ))) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/watermark_filter.rs b/src/stream/src/from_proto/watermark_filter.rs index cc3382bcc65fc..f01695e991487 100644 --- a/src/stream/src/from_proto/watermark_filter.rs +++ b/src/stream/src/from_proto/watermark_filter.rs @@ -34,7 +34,7 @@ impl ExecutorBuilder for WatermarkFilterBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let watermark_descs = node.get_watermark_descs().clone(); let [watermark_desc]: [_; 1] = watermark_descs.try_into().unwrap(); @@ -63,15 +63,15 @@ impl ExecutorBuilder for WatermarkFilterBuilder { let table = StateTable::from_table_catalog_inconsistent_op(&table, store, Some(vnodes)).await; - Ok(WatermarkFilterExecutor::new( + let exec = WatermarkFilterExecutor::new( params.actor_context, - params.info, + ¶ms.info, input, watermark_expr, event_time_col_idx, table, global_watermark_table, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs index 5a2bde99da491..e4cbfd62a900a 100644 --- a/src/stream/src/task/stream_manager.rs +++ b/src/stream/src/task/stream_manager.rs @@ -44,7 +44,10 @@ use super::{unique_executor_id, unique_operator_id, BarrierCompleteResult}; use crate::error::StreamResult; use crate::executor::monitor::StreamingMetrics; use crate::executor::subtask::SubtaskHandle; -use crate::executor::*; +use crate::executor::{ + Actor, ActorContext, ActorContextRef, Barrier, DispatchExecutor, DispatcherImpl, Executor, + ExecutorInfo, WrapperExecutor, +}; use crate::from_proto::create_executor; use crate::task::barrier_manager::{LocalBarrierEvent, LocalBarrierWorker}; use crate::task::{ @@ -102,7 +105,7 @@ pub struct ExecutorParams { pub op_info: String, /// The input executor. - pub input: Vec, + pub input: Vec, /// FragmentId of the actor pub fragment_id: FragmentId, @@ -348,7 +351,7 @@ impl StreamActorManager { /// Create dispatchers with downstream information registered before fn create_dispatcher( &self, - input: BoxedExecutor, + input: Executor, dispatchers: &[stream_plan::Dispatcher], actor_id: ActorId, fragment_id: FragmentId, @@ -381,7 +384,7 @@ impl StreamActorManager { vnode_bitmap: Option, has_stateful: bool, subtasks: &mut Vec, - ) -> StreamResult { + ) -> StreamResult { // The "stateful" here means that the executor may issue read operations to the state store // massively and continuously. Used to decide whether to apply the optimization of subtasks. fn is_stateful_executor(stream_node: &StreamNode) -> bool { @@ -432,21 +435,22 @@ impl StreamActorManager { let schema: Schema = node.fields.iter().map(Field::from).collect(); let identity = format!("{} {:X}", node.get_node_body().unwrap(), executor_id); + let info = ExecutorInfo { + schema, + pk_indices, + identity, + }; + let eval_error_report = ActorEvalErrorReport { actor_context: actor_context.clone(), - identity: identity.clone().into(), + identity: info.identity.clone().into(), }; // Build the executor with params. let executor_params = ExecutorParams { env: env.clone(), - info: ExecutorInfo { - schema: schema.clone(), - pk_indices: pk_indices.clone(), - identity: identity.clone(), - }, - + info: info.clone(), executor_id, operator_id, op_info, @@ -462,26 +466,14 @@ impl StreamActorManager { }; let executor = create_executor(executor_params, node, store).await?; - assert_eq!( - executor.pk_indices(), - &pk_indices, - "`pk_indices` of {} not consistent with what derived by optimizer", - executor.identity() - ); - assert_eq!( - executor.schema(), - &schema, - "`schema` of {} not consistent with what derived by optimizer", - executor.identity() - ); // Wrap the executor for debug purpose. - let executor = WrapperExecutor::new( + let wrapped = WrapperExecutor::new( executor, actor_context.clone(), env.config().developer.enable_executor_row_count, - ) - .boxed(); + ); + let executor = (info, wrapped).into(); // If there're multiple stateful executors in this actor, we will wrap it into a subtask. let executor = if has_stateful && is_stateful { @@ -507,7 +499,7 @@ impl StreamActorManager { env: StreamEnvironment, actor_context: &ActorContextRef, vnode_bitmap: Option, - ) -> StreamResult<(BoxedExecutor, Vec)> { + ) -> StreamResult<(Executor, Vec)> { let mut subtasks = vec![]; let executor = dispatch_state_store!(env.state_store(), store, { From 5bdf4e1e093ea9a14c606b3d7e5b52d7f2fda982 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Thu, 22 Feb 2024 23:07:05 +0800 Subject: [PATCH 09/13] fix all test code Signed-off-by: Richard Chien --- src/compute/tests/cdc_tests.rs | 117 ++--- src/compute/tests/integration_tests.rs | 48 +- src/stream/benches/stream_hash_agg.rs | 11 +- .../src/executor/backfill/cdc/cdc_backfill.rs | 3 +- src/stream/src/executor/batch_query.rs | 16 +- src/stream/src/executor/chain.rs | 73 ++- .../src/executor/dedup/append_only_dedup.rs | 22 +- src/stream/src/executor/dispatch.rs | 3 +- src/stream/src/executor/dml.rs | 21 +- src/stream/src/executor/dynamic_filter.rs | 25 +- src/stream/src/executor/expand.rs | 30 +- src/stream/src/executor/filter.rs | 18 +- src/stream/src/executor/hash_join.rs | 24 +- src/stream/src/executor/hop_window.rs | 13 +- src/stream/src/executor/integration_tests.rs | 62 +-- src/stream/src/executor/lookup/sides.rs | 16 +- src/stream/src/executor/lookup/tests.rs | 64 ++- src/stream/src/executor/lookup_union.rs | 76 ++- src/stream/src/executor/merge.rs | 13 +- src/stream/src/executor/mview/materialize.rs | 324 ++++++------ src/stream/src/executor/now.rs | 25 +- src/stream/src/executor/project.rs | 24 +- src/stream/src/executor/receiver.rs | 12 +- src/stream/src/executor/row_id_gen.rs | 18 +- src/stream/src/executor/simple_agg.rs | 5 +- src/stream/src/executor/sink.rs | 97 ++-- src/stream/src/executor/sort.rs | 11 +- .../src/executor/source/source_executor.rs | 14 +- .../src/executor/stateless_simple_agg.rs | 48 +- src/stream/src/executor/test_utils.rs | 50 +- src/stream/src/executor/top_n/group_top_n.rs | 133 +++-- .../src/executor/top_n/top_n_appendonly.rs | 88 ++-- src/stream/src/executor/top_n/top_n_plain.rs | 465 ++++++++---------- src/stream/src/executor/values.rs | 10 +- src/stream/src/executor/watermark_filter.rs | 7 +- .../src/executor/wrapper/epoch_check.rs | 24 +- .../src/executor/wrapper/schema_check.rs | 11 +- .../src/executor/wrapper/update_check.rs | 21 +- src/stream/src/from_proto/now.rs | 2 +- .../integration_tests/eowc_over_window.rs | 16 +- .../tests/integration_tests/hash_agg.rs | 20 +- .../tests/integration_tests/hop_window.rs | 24 +- .../tests/integration_tests/over_window.rs | 14 +- .../tests/integration_tests/project_set.rs | 27 +- 44 files changed, 910 insertions(+), 1235 deletions(-) diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs index 124e4e2ec6e7a..583e35dc9b7f6 100644 --- a/src/compute/tests/cdc_tests.rs +++ b/src/compute/tests/cdc_tests.rs @@ -44,29 +44,22 @@ use risingwave_stream::error::StreamResult; use risingwave_stream::executor::monitor::StreamingMetrics; use risingwave_stream::executor::test_utils::MockSource; use risingwave_stream::executor::{ - expect_first_barrier, ActorContext, AddMutation, Barrier, BoxedExecutor as StreamBoxedExecutor, - BoxedMessageStream, CdcBackfillExecutor, Execute, ExecutorInfo, ExternalStorageTable, - MaterializeExecutor, Message, Mutation, PkIndices, PkIndicesRef, StreamExecutorError, + expect_first_barrier, ActorContext, AddMutation, Barrier, BoxedMessageStream, + CdcBackfillExecutor, Execute, Executor as StreamExecutor, ExecutorInfo, ExternalStorageTable, + MaterializeExecutor, Message, Mutation, StreamExecutorError, }; // mock upstream binlog offset starting from "1.binlog, pos=0" pub struct MockOffsetGenExecutor { - upstream: Option, - - info: ExecutorInfo, + upstream: Option, start_offset: u32, } impl MockOffsetGenExecutor { - pub fn new(upstream: StreamBoxedExecutor, schema: Schema, pk_indices: PkIndices) -> Self { + pub fn new(upstream: StreamExecutor) -> Self { Self { upstream: Some(upstream), - info: ExecutorInfo { - schema, - pk_indices, - identity: "MockOffsetGenExecutor".to_string(), - }, start_offset: 0, } } @@ -133,22 +126,6 @@ impl Execute for MockOffsetGenExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> &ExecutorInfo { - &self.info - } } #[tokio::test] @@ -156,21 +133,26 @@ async fn test_cdc_backfill() -> StreamResult<()> { use risingwave_common::types::DataType; let memory_state_store = MemoryStateStore::new(); - let table_id = TableId::new(1002); - let schema = Schema::new(vec![ - Field::unnamed(DataType::Jsonb), // payload - Field::unnamed(DataType::Varchar), // _rw_offset - ]); - let column_ids = vec![0.into(), 1.into()]; - - let pk_indices = vec![0]; - - let (mut tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); - let _actor_ctx = ActorContext::for_test(0x3a3a3a); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor( + Schema::new(vec![ + Field::unnamed(DataType::Jsonb), // payload + ]), + vec![0], + ); // mock upstream offset (start from "1.binlog, pos=0") for ingested chunks - let mock_offset_executor = - MockOffsetGenExecutor::new(Box::new(source), schema.clone(), pk_indices.clone()); + let mock_offset_executor = StreamExecutor::new( + ExecutorInfo { + schema: Schema::new(vec![ + Field::unnamed(DataType::Jsonb), // payload + Field::unnamed(DataType::Varchar), // _rw_offset + ]), + pk_indices: vec![0], + identity: "MockOffsetGenExecutor".to_string(), + }, + MockOffsetGenExecutor::new(source).boxed(), + ); let binlog_file = String::from("1.binlog"); // mock binlog watermarks for backfill @@ -190,13 +172,15 @@ async fn test_cdc_backfill() -> StreamResult<()> { Field::with_name(DataType::Int64, "id"), // primary key Field::with_name(DataType::Float64, "price"), ]); + let table_pk_indices = vec![0]; + let table_pk_order_types = vec![OrderType::ascending()]; let external_table = ExternalStorageTable::new( - table_id, + TableId::new(1234), table_name, ExternalTableReaderImpl::Mock(MockExternalTableReader::new(binlog_watermarks)), table_schema.clone(), - vec![OrderType::ascending()], - pk_indices, + table_pk_order_types, + table_pk_indices.clone(), vec![0, 1], ); @@ -226,32 +210,35 @@ async fn test_cdc_backfill() -> StreamResult<()> { vec![0_usize], ) .await; - let info = ExecutorInfo { - schema: table_schema.clone(), - pk_indices: vec![0], - identity: "CdcBackfillExecutor".to_string(), - }; - let cdc_backfill = CdcBackfillExecutor::new( - ActorContext::for_test(actor_id), - info, - external_table, - Box::new(mock_offset_executor), - vec![0, 1], - None, - Arc::new(StreamingMetrics::unused()), - state_table, - 4, // 4 rows in a snapshot chunk - false, + + let cdc_backfill = StreamExecutor::new( + ExecutorInfo { + schema: table_schema.clone(), + pk_indices: table_pk_indices, + identity: "CdcBackfillExecutor".to_string(), + }, + CdcBackfillExecutor::new( + ActorContext::for_test(actor_id), + external_table, + mock_offset_executor, + vec![0, 1], + None, + Arc::new(StreamingMetrics::unused()), + state_table, + 4, // 4 rows in a snapshot chunk + false, + ) + .boxed(), ); // Create a `MaterializeExecutor` to write the changes to storage. + let materialize_table_id = TableId::new(5678); // TODO() let mut materialize = MaterializeExecutor::for_test( - Box::new(cdc_backfill), + cdc_backfill, memory_state_store.clone(), - table_id, + materialize_table_id, vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids.clone(), - 4, + vec![0.into(), 1.into()], Arc::new(AtomicU64::new(0)), ConflictBehavior::Overwrite, ) @@ -356,7 +343,7 @@ async fn test_cdc_backfill() -> StreamResult<()> { // Since we have not polled `Materialize`, we cannot scan anything from this table let table = StorageTable::for_test( memory_state_store.clone(), - table_id, + materialize_table_id, column_descs.clone(), vec![OrderType::ascending()], vec![0], diff --git a/src/compute/tests/integration_tests.rs b/src/compute/tests/integration_tests.rs index e8b70669b32a4..5e9fb3dfea86a 100644 --- a/src/compute/tests/integration_tests.rs +++ b/src/compute/tests/integration_tests.rs @@ -56,7 +56,7 @@ use risingwave_stream::executor::monitor::StreamingMetrics; use risingwave_stream::executor::row_id_gen::RowIdGenExecutor; use risingwave_stream::executor::source_executor::SourceExecutor; use risingwave_stream::executor::{ - ActorContext, Barrier, Execute, ExecutorInfo, MaterializeExecutor, Message, PkIndices, + ActorContext, Barrier, Execute, Executor, ExecutorInfo, MaterializeExecutor, Message, PkIndices, }; use tokio::sync::mpsc::unbounded_channel; @@ -162,56 +162,58 @@ async fn test_table_materialize() -> StreamResult<()> { let system_params_manager = LocalSystemParamsManager::for_test(); // Create a `SourceExecutor` to read the changes. - let source_executor = SourceExecutor::::new( - actor_ctx.clone(), + let source_executor = Executor::new( ExecutorInfo { schema: all_schema.clone(), pk_indices: pk_indices.clone(), identity: format!("SourceExecutor {:X}", 1), }, - None, // There is no external stream source. - Arc::new(StreamingMetrics::unused()), - barrier_rx, - system_params_manager.get_params(), - SourceCtrlOpts::default(), - ConnectorParams::default(), + SourceExecutor::::new( + actor_ctx.clone(), + None, // There is no external stream source. + Arc::new(StreamingMetrics::unused()), + barrier_rx, + system_params_manager.get_params(), + SourceCtrlOpts::default(), + ConnectorParams::default(), + ) + .boxed(), ); // Create a `DmlExecutor` to accept data change from users. - let dml_executor = DmlExecutor::new( + let dml_executor = Executor::new( ExecutorInfo { schema: all_schema.clone(), pk_indices: pk_indices.clone(), identity: format!("DmlExecutor {:X}", 2), }, - Box::new(source_executor), - dml_manager.clone(), - table_id, - INITIAL_TABLE_VERSION_ID, - column_descs.clone(), - 1024, + DmlExecutor::new( + source_executor, + dml_manager.clone(), + table_id, + INITIAL_TABLE_VERSION_ID, + column_descs.clone(), + 1024, + ) + .boxed(), ); - let row_id_gen_executor = RowIdGenExecutor::new( - actor_ctx, + let row_id_gen_executor = Executor::new( ExecutorInfo { schema: all_schema.clone(), pk_indices: pk_indices.clone(), identity: format!("RowIdGenExecutor {:X}", 3), }, - Box::new(dml_executor), - row_id_index, - vnodes, + RowIdGenExecutor::new(actor_ctx, dml_executor, row_id_index, vnodes).boxed(), ); // Create a `MaterializeExecutor` to write the changes to storage. let mut materialize = MaterializeExecutor::for_test( - Box::new(row_id_gen_executor), + row_id_gen_executor, memory_state_store.clone(), table_id, vec![ColumnOrder::new(0, OrderType::ascending())], all_column_ids.clone(), - 4, Arc::new(AtomicU64::new(0)), ConflictBehavior::NoCheck, ) diff --git a/src/stream/benches/stream_hash_agg.rs b/src/stream/benches/stream_hash_agg.rs index 90ac7ef7725e7..e10cd2629a8bc 100644 --- a/src/stream/benches/stream_hash_agg.rs +++ b/src/stream/benches/stream_hash_agg.rs @@ -26,7 +26,7 @@ use risingwave_storage::memory::MemoryStateStore; use risingwave_storage::StateStore; use risingwave_stream::executor::test_utils::agg_executor::new_boxed_hash_agg_executor; use risingwave_stream::executor::test_utils::*; -use risingwave_stream::executor::{BoxedExecutor, PkIndices}; +use risingwave_stream::executor::{Executor, PkIndices}; use tokio::runtime::Runtime; risingwave_expr_impl::enable!(); @@ -47,7 +47,7 @@ fn bench_hash_agg(c: &mut Criterion) { /// This aims to mirror `q17`'s aggregator. /// We can include more executor patterns as needed. -fn setup_bench_hash_agg(store: S) -> BoxedExecutor { +fn setup_bench_hash_agg(store: S) -> Executor { // ---- Define hash agg executor parameters ---- let input_data_types = vec![ // to_char(date_time) @@ -119,7 +119,8 @@ fn setup_bench_hash_agg(store: S) -> BoxedExecutor { ); // ---- Create MockSourceExecutor ---- - let (mut tx, source) = MockSource::channel(schema, PkIndices::new()); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, PkIndices::new()); tx.push_barrier(1, false); for chunk in chunks { tx.push_chunk(chunk); @@ -134,7 +135,7 @@ fn setup_bench_hash_agg(store: S) -> BoxedExecutor { block_on(new_boxed_hash_agg_executor( store, - Box::new(source), + source, false, agg_calls, row_count_index, @@ -146,7 +147,7 @@ fn setup_bench_hash_agg(store: S) -> BoxedExecutor { )) } -pub async fn execute_executor(executor: BoxedExecutor) { +pub async fn execute_executor(executor: Executor) { let mut stream = executor.execute(); while let Some(ret) = stream.next().await { _ = black_box(ret.unwrap()); diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index c7a5180d42ecc..34dc60e5088e2 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -634,7 +634,8 @@ mod tests { Field::unnamed(DataType::Varchar), // _rw_table_name ]); let pk_indices = vec![1]; - let (mut tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema.clone(), pk_indices.clone()); // let payload = r#"{"before": null,"after":{"O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" },"source":{"version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null},"op":"r","ts_ms":1695277757017,"transaction":null}"#.to_string(); let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#; diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs index 3e2ac96f796cf..7c92bcd732423 100644 --- a/src/stream/src/executor/batch_query.rs +++ b/src/stream/src/executor/batch_query.rs @@ -88,9 +88,6 @@ where #[cfg(test)] mod test { - - use std::vec; - use futures_async_stream::for_await; use super::*; @@ -102,15 +99,10 @@ mod test { let test_batch_count = 5; let table = gen_basic_table(test_batch_count * test_batch_size).await; - let info = ExecutorInfo { - schema: table.schema().clone(), - pk_indices: vec![0, 1], - identity: "BatchQuery".to_owned(), - }; - - let executor = Box::new(BatchQueryExecutor::new(table, test_batch_size, info)); - - let stream = executor.execute_with_epoch(u64::MAX); + let schema = table.schema().clone(); + let stream = BatchQueryExecutor::new(table, test_batch_size, schema) + .boxed() + .execute_with_epoch(u64::MAX); let mut batch_cnt = 0; #[for_await] diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index d42da1995115c..fdd1daff02e87 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -118,9 +118,7 @@ mod test { use super::ChainExecutor; use crate::executor::test_utils::MockSource; - use crate::executor::{ - AddMutation, Barrier, Execute, ExecutorInfo, Message, Mutation, PkIndices, - }; + use crate::executor::{AddMutation, Barrier, Execute, Message, Mutation, PkIndices}; use crate::task::{CreateMviewProgress, LocalBarrierManager}; #[tokio::test] @@ -130,48 +128,35 @@ mod test { let actor_id = progress.actor_id(); let schema = Schema::new(vec![Field::unnamed(DataType::Int64)]); - let first = Box::new( - MockSource::with_chunks( - schema.clone(), - PkIndices::new(), - vec![ - StreamChunk::from_pretty("I\n + 1"), - StreamChunk::from_pretty("I\n + 2"), - ], - ) - .stop_on_finish(false), - ); - - let second = Box::new(MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1).with_mutation(Mutation::Add( - AddMutation { - adds: maplit::hashmap! { - 0 => vec![Dispatcher { - downstream_actor_id: vec![actor_id], - ..Default::default() - }], - }, - added_actors: maplit::hashset! { actor_id }, - splits: Default::default(), - pause: false, + let first = MockSource::with_chunks(vec![ + StreamChunk::from_pretty("I\n + 1"), + StreamChunk::from_pretty("I\n + 2"), + ]) + .stop_on_finish(false) + .to_executor(schema.clone(), PkIndices::new()); + + let second = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1).with_mutation(Mutation::Add( + AddMutation { + adds: maplit::hashmap! { + 0 => vec![Dispatcher { + downstream_actor_id: vec![actor_id], + ..Default::default() + }], }, - ))), - Message::Chunk(StreamChunk::from_pretty("I\n + 3")), - Message::Chunk(StreamChunk::from_pretty("I\n + 4")), - ], - )); - - let info = ExecutorInfo { - schema, - pk_indices: PkIndices::new(), - identity: "ChainExecutor".to_string(), - }; - let chain = ChainExecutor::new(info, first, second, progress, false); - - let mut chain = Box::new(chain).execute(); + added_actors: maplit::hashset! { actor_id }, + splits: Default::default(), + pause: false, + }, + ))), + Message::Chunk(StreamChunk::from_pretty("I\n + 3")), + Message::Chunk(StreamChunk::from_pretty("I\n + 4")), + ]) + .to_executor(schema.clone(), PkIndices::new()); + + let chain = ChainExecutor::new(first, second, progress, false); + + let mut chain = chain.boxed().execute(); chain.next().await; let mut count = 0; diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index 4154ab1bc45c4..e1e711207ff72 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -217,7 +217,8 @@ mod tests { Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), ]); - let pk_indices = vec![0]; + let dedup_col_indices = vec![0]; + let pk_indices = dedup_col_indices.clone(); let order_types = vec![OrderType::ascending()]; let state_store = MemoryStateStore::new(); @@ -230,20 +231,17 @@ mod tests { ) .await; - let (mut tx, input) = MockSource::channel(schema.clone(), pk_indices.clone()); - let info = ExecutorInfo { - schema, - pk_indices, - identity: "AppendOnlyDedupExecutor".to_string(), - }; - let mut dedup_executor = Box::new(AppendOnlyDedupExecutor::new( - Box::new(input), - state_table, - info, + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, pk_indices); + let mut dedup_executor = AppendOnlyDedupExecutor::new( ActorContext::for_test(123), + source, + dedup_col_indices, + state_table, Arc::new(AtomicU64::new(0)), Arc::new(StreamingMetrics::unused()), - )) + ) + .boxed() .execute(); tx.push_barrier(1, false); diff --git a/src/stream/src/executor/dispatch.rs b/src/stream/src/executor/dispatch.rs index 8a6476afff910..ee347ee489258 100644 --- a/src/stream/src/executor/dispatch.rs +++ b/src/stream/src/executor/dispatch.rs @@ -1042,6 +1042,7 @@ mod tests { use crate::executor::exchange::output::Output; use crate::executor::exchange::permit::channel_for_test; use crate::executor::receiver::ReceiverExecutor; + use crate::executor::Execute; use crate::task::test_utils::helper_make_local_actor; #[derive(Debug)] @@ -1152,7 +1153,7 @@ mod tests { let (tx, rx) = channel_for_test(); let actor_id = 233; let fragment_id = 666; - let input = Box::new(ReceiverExecutor::for_test(rx)); + let input = Executor::new(Default::default(), ReceiverExecutor::for_test(rx).boxed()); let ctx = Arc::new(SharedContext::for_test()); let metrics = Arc::new(StreamingMetrics::unused()); diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index 9ac93f0368872..e25340f811aab 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -25,9 +25,7 @@ use risingwave_common::transaction::transaction_message::TxnMsg; use risingwave_dml::dml_manager::DmlManagerRef; use super::error::StreamExecutorError; -use super::{ - expect_first_barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, Mutation, -}; +use super::{expect_first_barrier, BoxedMessageStream, Execute, Executor, Message, Mutation}; use crate::common::StreamChunkBuilder; use crate::executor::stream_reader::StreamReaderWithPause; @@ -308,23 +306,18 @@ mod tests { let pk_indices = vec![0]; let dml_manager = Arc::new(DmlManager::for_test()); - let (mut tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); - let info = ExecutorInfo { - schema, - pk_indices, - identity: "DmlExecutor".to_string(), - }; + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, pk_indices); - let dml_executor = Box::new(DmlExecutor::new( - info, - Box::new(source), + let dml_executor = DmlExecutor::new( + source, dml_manager.clone(), table_id, INITIAL_TABLE_VERSION_ID, column_descs, 1024, - )); - let mut dml_executor = dml_executor.execute(); + ); + let mut dml_executor = dml_executor.boxed().execute(); let stream_chunk1 = StreamChunk::from_pretty( " I I diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index d34fd612c966b..6c17c380c02fe 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -550,21 +550,20 @@ mod tests { let schema = Schema { fields: vec![Field::unnamed(DataType::Int64)], }; - let (tx_l, source_l) = MockSource::channel(schema.clone(), vec![0]); - let (tx_r, source_r) = MockSource::channel(schema, vec![]); - - let schema = source_l.schema().clone(); - let info = ExecutorInfo { - schema, - pk_indices: vec![0], - identity: "DynamicFilterExecutor".to_string(), - }; + let (tx_l, source_l) = MockSource::channel(); + let source_l = source_l.to_executor(schema.clone(), vec![0]); + let (tx_r, source_r) = MockSource::channel(); + let source_r = source_r.to_executor(schema, vec![]); let executor = DynamicFilterExecutor::::new( ActorContext::for_test(123), - info, - Box::new(source_l), - Box::new(source_r), + &ExecutorInfo { + schema: source_l.schema().clone(), + pk_indices: vec![0], + identity: "DynamicFilterExecutor".to_string(), + }, + source_l, + source_r, 0, comparator, mem_state_l, @@ -574,7 +573,7 @@ mod tests { always_relax, false, ); - (tx_l, tx_r, Box::new(executor).execute()) + (tx_l, tx_r, executor.boxed().execute()) } #[tokio::test] diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index 32546dc45d6fe..5e5672b035bce 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -19,7 +19,7 @@ use futures_async_stream::try_stream; use risingwave_common::array::{Array, I64Array, StreamChunk}; use super::error::StreamExecutorError; -use super::{BoxedMessageStream, Execute, Executor, ExecutorInfo, Message}; +use super::{BoxedMessageStream, Execute, Executor, Message}; pub struct ExpandExecutor { input: Executor, @@ -81,7 +81,7 @@ mod tests { use super::ExpandExecutor; use crate::executor::test_utils::MockSource; - use crate::executor::{Execute, ExecutorInfo, PkIndices}; + use crate::executor::{Execute, PkIndices}; #[tokio::test] async fn test_expand() { @@ -92,29 +92,19 @@ mod tests { + 6 6 3 - 7 5 4", ); - let input_schema = Schema { - fields: vec![ + let source = MockSource::with_chunks(vec![chunk1]).to_executor( + Schema::new(vec![ Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), - ], - }; - let source = MockSource::with_chunks(input_schema.clone(), PkIndices::new(), vec![chunk1]); - let schema = { - let mut fields = input_schema.into_fields(); - fields.extend(fields.clone()); - fields.push(Field::with_name(DataType::Int64, "flag")); - Schema::new(fields) - }; - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "ExpandExecutor".to_string(), - }; + ]), + PkIndices::new(), + ); let column_subsets = vec![vec![0, 1], vec![1, 2]]; - let expand = Box::new(ExpandExecutor::new(info, Box::new(source), column_subsets)); - let mut expand = expand.execute(); + let mut expand = ExpandExecutor::new(source, column_subsets) + .boxed() + .execute(); let chunk = expand.next().await.unwrap().unwrap().into_chunk().unwrap(); assert_eq!( diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index 6ec6091eaa3c7..a3b908ec53797 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -23,8 +23,8 @@ use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; use super::{ - ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, - StreamExecutorError, StreamExecutorResult, + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, StreamExecutorError, + StreamExecutorResult, }; /// `FilterExecutor` filters data with the `expr`. The `expr` takes a chunk of data, @@ -203,8 +203,8 @@ mod tests { ], }; let pk_indices = PkIndices::new(); - let source = - MockSource::with_chunks(schema.clone(), pk_indices.clone(), vec![chunk1, chunk2]); + let source = MockSource::with_chunks(vec![chunk1, chunk2]) + .to_executor(schema.clone(), pk_indices.clone()); let info = ExecutorInfo { schema, pk_indices, @@ -213,13 +213,9 @@ mod tests { let test_expr = build_from_pretty("(greater_than:boolean $0:int8 $1:int8)"); - let filter = Box::new(FilterExecutor::new( - ActorContext::for_test(123), - info, - Box::new(source), - test_expr, - )); - let mut filter = filter.execute(); + let mut filter = FilterExecutor::new(ActorContext::for_test(123), source, test_expr) + .boxed() + .execute(); let chunk = filter.next().await.unwrap().unwrap().into_chunk().unwrap(); assert_eq!( diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index aa3477ca5cb11..3c3728d3eb428 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -1142,8 +1142,10 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (tx_l, source_l) = MockSource::channel(schema.clone(), vec![1]); - let (tx_r, source_r) = MockSource::channel(schema, vec![1]); + let (tx_l, source_l) = MockSource::channel(); + let source_l = source_l.to_executor(schema.clone(), vec![1]); + let (tx_r, source_r) = MockSource::channel(); + let source_r = source_r.to_executor(schema, vec![1]); let params_l = JoinParams::new(vec![0], vec![1]); let params_r = JoinParams::new(vec![0], vec![1]); let cond = with_condition.then(|| create_cond(condition_text)); @@ -1186,8 +1188,8 @@ mod tests { let executor = HashJoinExecutor::::new( ActorContext::for_test(123), info, - Box::new(source_l), - Box::new(source_r), + source_l, + source_r, params_l, params_r, vec![null_safe], @@ -1203,7 +1205,7 @@ mod tests { Arc::new(StreamingMetrics::unused()), 1024, ); - (tx_l, tx_r, Box::new(executor).execute()) + (tx_l, tx_r, executor.boxed().execute()) } async fn create_classical_executor( @@ -1224,8 +1226,10 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (tx_l, source_l) = MockSource::channel(schema.clone(), vec![0]); - let (tx_r, source_r) = MockSource::channel(schema, vec![0]); + let (tx_l, source_l) = MockSource::channel(); + let source_l = source_l.to_executor(schema.clone(), vec![0]); + let (tx_r, source_r) = MockSource::channel(); + let source_r = source_r.to_executor(schema, vec![0]); let params_l = JoinParams::new(vec![0, 1], vec![]); let params_r = JoinParams::new(vec![0, 1], vec![]); let cond = with_condition.then(|| create_cond(None)); @@ -1276,8 +1280,8 @@ mod tests { let executor = HashJoinExecutor::::new( ActorContext::for_test(123), info, - Box::new(source_l), - Box::new(source_r), + source_l, + source_r, params_l, params_r, vec![false], @@ -1293,7 +1297,7 @@ mod tests { Arc::new(StreamingMetrics::unused()), 1024, ); - (tx_l, tx_r, Box::new(executor).execute()) + (tx_l, tx_r, executor.boxed().execute()) } #[tokio::test] diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs index 1e928c929ae67..a839d0075f372 100644 --- a/src/stream/src/executor/hop_window.rs +++ b/src/stream/src/executor/hop_window.rs @@ -238,8 +238,9 @@ mod tests { use risingwave_expr::expr::test_utils::make_hop_window_expression; use risingwave_expr::expr::NonStrictExpression; + use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Execute, ExecutorInfo, StreamChunk}; + use crate::executor::{ActorContext, Execute, StreamChunk}; const CHUNK_SIZE: usize = 256; @@ -263,7 +264,7 @@ mod tests { .replace('^', "2022-02-02T"), ); let input = - MockSource::with_chunks(schema.clone(), pk_indices.clone(), vec![chunk]).boxed(); + MockSource::with_chunks(vec![chunk]).to_executor(schema.clone(), pk_indices.clone()); let window_slide = Interval::from_minutes(15); let window_size = Interval::from_minutes(30); let window_offset = Interval::from_minutes(0); @@ -276,14 +277,8 @@ mod tests { ) .unwrap(); - super::HopWindowExecutor::new( + HopWindowExecutor::new( ActorContext::for_test(123), - ExecutorInfo { - // TODO: the schema is incorrect, but it seems useless here. - schema, - pk_indices, - identity: "HopWindowExecutor".to_string(), - }, input, 2, window_slide, diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs index 258709955776f..691ba0f46f341 100644 --- a/src/stream/src/executor/integration_tests.rs +++ b/src/stream/src/executor/integration_tests.rs @@ -50,27 +50,25 @@ async fn test_merger_sum_aggr() { let actor_ctx = ActorContext::for_test(0); // `make_actor` build an actor to do local aggregation let make_actor = |input_rx| { - let _schema = Schema { + let input_schema = Schema { fields: vec![Field::unnamed(DataType::Int64)], }; - let input = ReceiverExecutor::for_test(input_rx); + let input = Executor::new( + ExecutorInfo { + schema: input_schema, + pk_indices: PkIndices::new(), + identity: "ReceiverExecutor".to_string(), + }, + ReceiverExecutor::for_test(input_rx).boxed(), + ); let agg_calls = vec![ AggCall::from_pretty("(count:int8)"), AggCall::from_pretty("(sum:int8 $0:int8)"), ]; let schema = generate_agg_schema(&input, &agg_calls, None); // for the local aggregator, we need two states: row count and sum - let aggregator = StatelessSimpleAggExecutor::new( - actor_ctx.clone(), - ExecutorInfo { - schema, - pk_indices: vec![], - identity: format!("StatelessSimpleAggExecutor {:X}", 1), - }, - input.boxed(), - agg_calls, - ) - .unwrap(); + let aggregator = + StatelessSimpleAggExecutor::new(actor_ctx.clone(), input, schema, agg_calls).unwrap(); let (tx, rx) = channel_for_test(); let consumer = SenderConsumer { input: aggregator.boxed(), @@ -108,13 +106,15 @@ async fn test_merger_sum_aggr() { // create a round robin dispatcher, which dispatches messages to the actors let (input, rx) = channel_for_test(); - let schema = Schema { - fields: vec![ - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - ], - }; - let receiver_op = Box::new(ReceiverExecutor::for_test(rx)); + let receiver_op = Executor::new( + ExecutorInfo { + // input schema of local simple agg + schema: Schema::new(vec![Field::unnamed(DataType::Int64)]), + pk_indices: PkIndices::new(), + identity: "ReceiverExecutor".to_string(), + }, + ReceiverExecutor::for_test(rx).boxed(), + ); let dispatcher = DispatchExecutor::new( receiver_op, vec![DispatcherImpl::RoundRobin(RoundRobinDataDispatcher::new( @@ -138,14 +138,25 @@ async fn test_merger_sum_aggr() { handles.push(tokio::spawn(actor.run())); // use a merge operator to collect data from dispatchers before sending them to aggregator - let merger = MergeExecutor::for_test(outputs, schema); + let merger = Executor::new( + ExecutorInfo { + // output schema of local simple agg + schema: Schema::new(vec![ + Field::unnamed(DataType::Int64), + Field::unnamed(DataType::Int64), + ]), + pk_indices: PkIndices::new(), + identity: "MergeExecutor".to_string(), + }, + MergeExecutor::for_test(outputs).boxed(), + ); // for global aggregator, we need to sum data and sum row count let is_append_only = false; let aggregator = new_boxed_simple_agg_executor( actor_ctx.clone(), MemoryStateStore::new(), - merger.boxed(), + merger, is_append_only, vec![ AggCall::from_pretty("(sum0:int8 $0:int8)"), @@ -160,13 +171,6 @@ async fn test_merger_sum_aggr() { let projection = ProjectExecutor::new( actor_ctx.clone(), - ExecutorInfo { - schema: Schema { - fields: vec![Field::unnamed(DataType::Int64)], - }, - pk_indices: vec![], - identity: format!("ProjectExecutor {:X}", 3), - }, aggregator, vec![ // TODO: use the new streaming_if_null expression here, and add `None` tests diff --git a/src/stream/src/executor/lookup/sides.rs b/src/stream/src/executor/lookup/sides.rs index df395418b7ed5..2c41eae40bd47 100644 --- a/src/stream/src/executor/lookup/sides.rs +++ b/src/stream/src/executor/lookup/sides.rs @@ -426,14 +426,14 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (mut tx_l, source_l) = MockSource::channel(schema.clone(), vec![1]); - let (tx_r, source_r) = MockSource::channel(schema, vec![1]); - - let mut stream = stream_lookup_arrange_this_epoch( - Box::new(source_l.stop_on_finish(false)), - Box::new(source_r.stop_on_finish(false)), - ) - .boxed(); + let (mut tx_l, source_l) = MockSource::channel(); + let source_l = source_l + .stop_on_finish(false) + .to_executor(schema.clone(), vec![1]); + let (tx_r, source_r) = MockSource::channel(); + let source_r = source_r.stop_on_finish(false).to_executor(schema, vec![1]); + + let mut stream = stream_lookup_arrange_this_epoch(source_l, source_r).boxed(); // Simulate recovery test drop(tx_r); diff --git a/src/stream/src/executor/lookup/tests.rs b/src/stream/src/executor/lookup/tests.rs index 9f55bd30fb479..4cda86614dc12 100644 --- a/src/stream/src/executor/lookup/tests.rs +++ b/src/stream/src/executor/lookup/tests.rs @@ -30,8 +30,8 @@ use crate::executor::lookup::impl_::LookupExecutorParams; use crate::executor::lookup::LookupExecutor; use crate::executor::test_utils::*; use crate::executor::{ - ActorContext, Barrier, BoxedMessageStream, Execute, ExecutorInfo, MaterializeExecutor, Message, - PkIndices, + ActorContext, Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, + MaterializeExecutor, Message, PkIndices, }; fn arrangement_col_descs() -> Vec { @@ -68,10 +68,7 @@ fn arrangement_col_arrange_rules_join_key() -> Vec { /// | + | 2337 | 8 | 3 | /// | - | 2333 | 6 | 3 | /// | b | | | 3 -> 4 | -async fn create_arrangement( - table_id: TableId, - memory_state_store: MemoryStateStore, -) -> Box { +async fn create_arrangement(table_id: TableId, memory_state_store: MemoryStateStore) -> Executor { // Two columns of int32 type, the second column is arrange key. let columns = arrangement_col_descs(); @@ -101,30 +98,32 @@ async fn create_arrangement( .collect_vec(), ); - let source = MockSource::with_messages( - schema, - vec![0], - vec![ - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(4)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(4)), + ]) + .to_executor(schema, vec![0]); - Box::new( + Executor::new( + ExecutorInfo { + schema: source.schema().clone(), + pk_indices: source.pk_indices().to_vec(), + identity: "MaterializeExecutor".to_string(), + }, MaterializeExecutor::for_test( - Box::new(source), + source, memory_state_store, table_id, arrangement_col_arrange_rules(), column_ids, - 1, Arc::new(AtomicU64::new(0)), ConflictBehavior::NoCheck, ) - .await, + .await + .boxed(), ) } @@ -139,7 +138,7 @@ async fn create_arrangement( /// | b | | | 2 -> 3 | /// | - | 6 | 1 | 3 | /// | b | | | 3 -> 4 | -fn create_source() -> Box { +fn create_source() -> Executor { let columns = vec![ ColumnDesc::new_atomic(DataType::Int64, "join_column", 1), ColumnDesc::new_atomic(DataType::Int64, "rowid_column", 2), @@ -163,19 +162,16 @@ fn create_source() -> Box { .collect_vec(), ); - let source = MockSource::with_messages( - schema, - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(4)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(4)), + ]) + .to_executor(schema, PkIndices::new()); - Box::new(source) + source } async fn next_msg(buffer: &mut Vec, executor: &mut BoxedMessageStream) { diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index 238319ee9e396..7722582f8194a 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -20,7 +20,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use super::error::StreamExecutorError; -use super::{Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message}; +use super::{Barrier, BoxedMessageStream, Execute, Executor, Message}; /// Merges data from multiple inputs with order. If `order = [2, 1, 0]`, then /// it will first pipe data from the third input; after the third input gets a barrier, it will then @@ -139,52 +139,36 @@ mod tests { let schema = Schema { fields: vec![Field::unnamed(DataType::Int64)], }; - let source0 = MockSource::with_messages( - schema.clone(), - vec![0], - vec![ - Message::Chunk(StreamChunk::from_pretty("I\n + 1")), - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(StreamChunk::from_pretty("I\n + 2")), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(StreamChunk::from_pretty("I\n + 3")), - Message::Barrier(Barrier::new_test_barrier(3)), - ], - ) - .stop_on_finish(false); - let source1 = MockSource::with_messages( - schema.clone(), - vec![0], - vec![ - Message::Chunk(StreamChunk::from_pretty("I\n + 11")), - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(StreamChunk::from_pretty("I\n + 12")), - Message::Barrier(Barrier::new_test_barrier(2)), - ], - ) - .stop_on_finish(false); - let source2 = MockSource::with_messages( - schema, - vec![0], - vec![ - Message::Chunk(StreamChunk::from_pretty("I\n + 21")), - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(StreamChunk::from_pretty("I\n + 22")), - Message::Barrier(Barrier::new_test_barrier(2)), - ], - ) - .stop_on_finish(false); + let source0 = MockSource::with_messages(vec![ + Message::Chunk(StreamChunk::from_pretty("I\n + 1")), + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(StreamChunk::from_pretty("I\n + 2")), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(StreamChunk::from_pretty("I\n + 3")), + Message::Barrier(Barrier::new_test_barrier(3)), + ]) + .stop_on_finish(false) + .to_executor(schema.clone(), vec![0]); + let source1 = MockSource::with_messages(vec![ + Message::Chunk(StreamChunk::from_pretty("I\n + 11")), + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(StreamChunk::from_pretty("I\n + 12")), + Message::Barrier(Barrier::new_test_barrier(2)), + ]) + .stop_on_finish(false) + .to_executor(schema.clone(), vec![0]); + let source2 = MockSource::with_messages(vec![ + Message::Chunk(StreamChunk::from_pretty("I\n + 21")), + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(StreamChunk::from_pretty("I\n + 22")), + Message::Barrier(Barrier::new_test_barrier(2)), + ]) + .stop_on_finish(false) + .to_executor(schema, vec![0]); - let executor = Box::new(LookupUnionExecutor::new( - ExecutorInfo { - schema: source0.schema().clone(), - pk_indices: vec![0], - identity: "LookupUnionExecutor".to_string(), - }, - vec![Box::new(source0), Box::new(source1), Box::new(source2)], - vec![2, 1, 0], - )) - .execute(); + let executor = LookupUnionExecutor::new(vec![source0, source1, source2], vec![2, 1, 0]) + .boxed() + .execute(); let outputs: Vec<_> = executor.try_collect().await.unwrap(); assert_eq!( diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index 16097f1cd1ffb..c6fd4232ea232 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -75,7 +75,7 @@ impl MergeExecutor { } #[cfg(test)] - pub fn for_test(inputs: Vec, schema: Schema) -> Self { + pub fn for_test(inputs: Vec) -> Self { use super::exchange::input::LocalInput; use crate::executor::exchange::input::Input; @@ -460,7 +460,7 @@ mod tests { txs.push(tx); rxs.push(rx); } - let merger = MergeExecutor::for_test(rxs, Schema::default()); + let merger = MergeExecutor::for_test(rxs); let mut handles = Vec::with_capacity(CHANNEL_NUMBER); let epochs = (10..1000u64).step_by(10).collect_vec(); @@ -569,13 +569,8 @@ mod tests { .try_collect() .unwrap(); - let merge = MergeExecutor::new( + let mut merge = MergeExecutor::new( ActorContext::for_test(actor_id), - ExecutorInfo { - schema, - pk_indices: vec![], - identity: "MergeExecutor".to_string(), - }, fragment_id, upstream_fragment_id, inputs, @@ -586,8 +581,6 @@ mod tests { .boxed() .execute(); - pin_mut!(merge); - // 2. Take downstream receivers. let txs = [untouched, old, new] .into_iter() diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index 050c6e2beb3f9..28a494d59411a 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -265,12 +265,10 @@ impl MaterializeExecutor { #[allow(clippy::too_many_arguments)] pub async fn for_test( input: Executor, - schema: Schema, store: S, table_id: TableId, keys: Vec, column_ids: Vec, - executor_id: u64, watermark_epoch: AtomicU64Ref, conflict_behavior: ConflictBehavior, ) -> Self { @@ -658,17 +656,14 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(3)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(3)), + ]) + .to_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -685,19 +680,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::NoCheck, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::NoCheck, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -766,17 +759,14 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(3)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(3)), + ]) + .to_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -793,19 +783,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::Overwrite, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::Overwrite, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -862,18 +850,15 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(chunk1), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(3)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(chunk1), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(3)), + ]) + .to_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -890,19 +875,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::Overwrite, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::Overwrite, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -994,19 +977,16 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(4)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(4)), + ]) + .to_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1023,19 +1003,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::Overwrite, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::Overwrite, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -1178,18 +1156,15 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(chunk1), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(3)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(chunk1), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(3)), + ]) + .to_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1206,19 +1181,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::IgnoreConflict, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::IgnoreConflict, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -1289,15 +1262,12 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(2)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(2)), + ]) + .to_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1314,19 +1284,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::IgnoreConflict, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::IgnoreConflict, ) + .await + .boxed() .execute(); let _msg1 = materialize_executor .next() @@ -1408,19 +1376,16 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(4)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(4)), + ]) + .to_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1437,19 +1402,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::IgnoreConflict, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::IgnoreConflict, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -1617,21 +1580,20 @@ mod tests { .chain(iter::once(Message::Barrier(Barrier::new_test_barrier(2)))) .collect(); // Prepare stream executors. - let source = MockSource::with_messages(schema.clone(), PkIndices::new(), messages); - - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store.clone(), - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - conflict_behavior, - ) - .await, + let source = + MockSource::with_messages(messages).to_executor(schema.clone(), PkIndices::new()); + + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store.clone(), + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + conflict_behavior, ) + .await + .boxed() .execute(); materialize_executor.expect_barrier().await; diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index f04447a563329..be429380d598d 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -158,7 +158,7 @@ impl Execute for NowExecutor { #[cfg(test)] mod tests { use risingwave_common::array::StreamChunk; - use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableId}; + use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId}; use risingwave_common::test_prelude::StreamChunkTestExt; use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_storage::memory::MemoryStateStore; @@ -168,8 +168,7 @@ mod tests { use crate::common::table::state_table::StateTable; use crate::executor::test_utils::StreamExecutorTestExt; use crate::executor::{ - Barrier, BoxedMessageStream, Execute, ExecutorInfo, Mutation, StreamExecutorResult, - Watermark, + Barrier, BoxedMessageStream, Execute, Mutation, StreamExecutorResult, Watermark, }; #[tokio::test] @@ -392,22 +391,8 @@ mod tests { let (sender, barrier_receiver) = unbounded_channel(); - let schema = Schema::new(vec![Field { - data_type: DataType::Timestamptz, - name: String::from("now"), - sub_fields: vec![], - type_name: String::default(), - }]); - - let now_executor = NowExecutor::new( - ExecutorInfo { - schema, - pk_indices: vec![], - identity: "NowExecutor".to_string(), - }, - barrier_receiver, - state_table, - ); - (sender, Box::new(now_executor).execute()) + let now_executor = + NowExecutor::new(vec![DataType::Timestamptz], barrier_receiver, state_table); + (sender, now_executor.boxed().execute()) } } diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index edc012572c2dd..cef4f6b997350 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -230,7 +230,8 @@ mod tests { ], }; let pk_indices = vec![0]; - let (mut tx, source) = MockSource::channel(schema, pk_indices); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, pk_indices); let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)"); @@ -242,16 +243,15 @@ mod tests { identity: "ProjectExecutor".to_string(), }; - let project = Box::new(ProjectExecutor::new( + let project = ProjectExecutor::new( ActorContext::for_test(123), - info, - Box::new(source), + source, vec![test_expr], MultiMap::new(), vec![], 0.0, - )); - let mut project = project.execute(); + ); + let mut project = project.boxed().execute(); tx.push_barrier(1, false); let barrier = project.next().await.unwrap().unwrap(); @@ -318,7 +318,8 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (mut tx, source) = MockSource::channel(schema, PkIndices::new()); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, PkIndices::new()); let a_expr = build_from_pretty("(add:int8 $0:int8 1:int8)"); let b_expr = build_from_pretty("(subtract:int8 $0:int8 1:int8)"); @@ -336,16 +337,15 @@ mod tests { identity: "ProjectExecutor".to_string(), }; - let project = Box::new(ProjectExecutor::new( + let project = ProjectExecutor::new( ActorContext::for_test(123), - info, - Box::new(source), + source, vec![a_expr, b_expr, c_expr], MultiMap::from_iter(vec![(0, 0), (0, 1)].into_iter()), vec![2], 0.0, - )); - let mut project = project.execute(); + ); + let mut project = project.boxed().execute(); tx.push_barrier(1, false); tx.push_int64_watermark(0, 100); diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index 8df2d988fca7d..a642a581ec653 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -24,7 +24,7 @@ use super::ActorContextRef; use crate::executor::exchange::input::new_input; use crate::executor::monitor::StreamingMetrics; use crate::executor::utils::ActorInputMetrics; -use crate::executor::{expect_first_barrier, BoxedMessageStream, Execute, ExecutorInfo, Message}; +use crate::executor::{expect_first_barrier, BoxedMessageStream, Execute, Message}; use crate::task::{FragmentId, SharedContext}; /// `ReceiverExecutor` is used along with a channel. After creating a mpsc channel, /// there should be a `ReceiverExecutor` running in the background, so as to push @@ -204,7 +204,6 @@ mod tests { use futures::{pin_mut, FutureExt}; use risingwave_common::array::StreamChunk; - use risingwave_common::catalog::Schema; use risingwave_pb::stream_plan::update_mutation::MergeUpdate; use super::*; @@ -213,8 +212,6 @@ mod tests { #[tokio::test] async fn test_configuration_change() { - let schema = Schema { fields: vec![] }; - let actor_id = 233; let (old, new) = (114, 514); // old and new upstream actor id @@ -244,15 +241,8 @@ mod tests { ) .unwrap(); - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "ReceiverExecutor".to_string(), - }; - let receiver = ReceiverExecutor::new( ActorContext::for_test(actor_id), - info, fragment_id, upstream_fragment_id, input, diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index d645221bdbcf5..49c7e02a1b1ad 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -147,20 +147,16 @@ mod tests { let pk_indices = vec![0]; let row_id_index = 0; let row_id_generator = Bitmap::ones(VirtualNode::COUNT); - let (mut tx, upstream) = MockSource::channel(schema.clone(), pk_indices.clone()); - let row_id_gen_executor = Box::new(RowIdGenExecutor::new( + let (mut tx, upstream) = MockSource::channel(); + let upstream = upstream.to_executor(schema.clone(), pk_indices.clone()); + + let row_id_gen_executor = RowIdGenExecutor::new( ActorContext::for_test(233), - ExecutorInfo { - schema, - pk_indices, - identity: "RowIdGenExecutor".to_string(), - }, - Box::new(upstream), + upstream, row_id_index, row_id_generator, - )); - - let mut row_id_gen_executor = row_id_gen_executor.execute(); + ); + let mut row_id_gen_executor = row_id_gen_executor.boxed().execute(); // Init barrier tx.push_barrier(1, false); diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index 90e8244aaa9b4..0eedd73125aa4 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -323,7 +323,8 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![2]); tx.push_barrier(1, false); tx.push_barrier(2, false); tx.push_chunk(StreamChunk::from_pretty( @@ -352,7 +353,7 @@ mod tests { let simple_agg = new_boxed_simple_agg_executor( ActorContext::for_test(123), store, - Box::new(source), + source, false, agg_calls, 0, diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index 8e81006e4133a..54b4cfae42fc7 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -463,28 +463,25 @@ mod test { .collect(); let pk_indices = vec![0]; - let mock = MockSource::with_messages( - schema.clone(), - pk_indices.clone(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 3 2 1", - ))), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I U- 3 2 1 U+ 3 4 1 + 5 6 7", - ))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I - 5 6 7", - ))), - ], - ); + ))), + ]) + .to_executor(schema.clone(), pk_indices.clone()); let sink_param = SinkParam { sink_id: 0.into(), @@ -510,7 +507,7 @@ mod test { let sink_executor = SinkExecutor::new( ActorContext::for_test(0), info, - Box::new(mock), + source, SinkWriterParam::for_test(), sink_param, columns.clone(), @@ -519,7 +516,7 @@ mod test { .await .unwrap(); - let mut executor = SinkExecutor::execute(Box::new(sink_executor)); + let mut executor = sink_executor.boxed().execute(); // Barrier message. executor.next().await.unwrap().unwrap(); @@ -586,32 +583,29 @@ mod test { .map(|column| Field::from(column.column_desc.clone())) .collect(); - let mock = MockSource::with_messages( - schema.clone(), - vec![0, 1], - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 1 1 10", - ))), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 1 3 30", - ))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 1 2 20 - 1 2 20", - ))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I - 1 1 10", - ))), - Message::Barrier(Barrier::new_test_barrier(3)), - ], - ); + ))), + Message::Barrier(Barrier::new_test_barrier(3)), + ]) + .to_executor(schema.clone(), vec![0, 1]); let sink_param = SinkParam { sink_id: 0.into(), @@ -637,7 +631,7 @@ mod test { let sink_executor = SinkExecutor::new( ActorContext::for_test(0), info, - Box::new(mock), + source, SinkWriterParam::for_test(), sink_param, columns.clone(), @@ -646,7 +640,7 @@ mod test { .await .unwrap(); - let mut executor = SinkExecutor::execute(Box::new(sink_executor)); + let mut executor = sink_executor.boxed().execute(); // Barrier message. executor.next().await.unwrap().unwrap(); @@ -727,15 +721,12 @@ mod test { .collect(); let pk_indices = vec![0]; - let mock = MockSource::with_messages( - schema.clone(), - pk_indices.clone(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Barrier(Barrier::new_test_barrier(3)), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Barrier(Barrier::new_test_barrier(3)), + ]) + .to_executor(schema.clone(), pk_indices.clone()); let sink_param = SinkParam { sink_id: 0.into(), @@ -761,7 +752,7 @@ mod test { let sink_executor = SinkExecutor::new( ActorContext::for_test(0), info, - Box::new(mock), + source, SinkWriterParam::for_test(), sink_param, columns, @@ -770,7 +761,7 @@ mod test { .await .unwrap(); - let mut executor = SinkExecutor::execute(Box::new(sink_executor)); + let mut executor = sink_executor.boxed().execute(); // Barrier message. assert_eq!( diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index 0153a7e7c7dd6..d9d257458306f 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -190,15 +190,12 @@ mod tests { ) .await; - let (tx, source) = MockSource::channel(input_schema, input_pk_indices); + let (tx, source) = MockSource::channel(); + let source = source.to_executor(input_schema, input_pk_indices); let sort_executor = SortExecutor::new(SortExecutorArgs { actor_ctx: ActorContext::for_test(123), - info: ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "SortExecutor".to_string(), - }, - input: source.boxed(), + schema: source.schema().clone(), + input: source, buffer_table, chunk_size: 1024, sort_column_index, diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index 9faf9f324efc9..026181fcfe3bd 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -731,11 +731,6 @@ mod tests { let executor = SourceExecutor::new( ActorContext::for_test(0), - ExecutorInfo { - schema, - pk_indices, - identity: "SourceExecutor".to_string(), - }, Some(core), Arc::new(StreamingMetrics::unused()), barrier_rx, @@ -743,7 +738,7 @@ mod tests { SourceCtrlOpts::default(), ConnectorParams::default(), ); - let mut executor = Box::new(executor).execute(); + let mut executor = executor.boxed().execute(); let init_barrier = Barrier::new_test_barrier(1).with_mutation(Mutation::Add(AddMutation { adds: HashMap::new(), @@ -825,11 +820,6 @@ mod tests { let executor = SourceExecutor::new( ActorContext::for_test(0), - ExecutorInfo { - schema, - pk_indices, - identity: "SourceExecutor".to_string(), - }, Some(core), Arc::new(StreamingMetrics::unused()), barrier_rx, @@ -837,7 +827,7 @@ mod tests { SourceCtrlOpts::default(), ConnectorParams::default(), ); - let mut handler = Box::new(executor).execute(); + let mut handler = executor.boxed().execute(); let init_barrier = Barrier::new_test_barrier(1).with_mutation(Mutation::Add(AddMutation { adds: HashMap::new(), diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index 586f2e0003f52..6ffdf5d25e565 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -142,29 +142,19 @@ mod tests { #[tokio::test] async fn test_no_chunk() { let schema = schema_test_utils::ii(); - let (mut tx, source) = MockSource::channel(schema, vec![2]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![2]); tx.push_barrier(1, false); tx.push_barrier(2, false); tx.push_barrier(3, false); let agg_calls = vec![AggCall::from_pretty("(count:int8)")]; let schema = generate_agg_schema(&source, &agg_calls, None); - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "StatelessSimpleAggExecutor".to_string(), - }; - - let simple_agg = Box::new( - StatelessSimpleAggExecutor::new( - ActorContext::for_test(123), - info, - Box::new(source), - agg_calls, - ) - .unwrap(), - ); - let mut simple_agg = simple_agg.execute(); + + let simple_agg = + StatelessSimpleAggExecutor::new(ActorContext::for_test(123), source, schema, agg_calls) + .unwrap(); + let mut simple_agg = simple_agg.boxed().execute(); assert_matches!( simple_agg.next().await.unwrap().unwrap(), @@ -183,7 +173,8 @@ mod tests { #[tokio::test] async fn test_local_simple_agg() { let schema = schema_test_utils::iii(); - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk\ + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![2]); tx.push_barrier(1, false); tx.push_chunk(StreamChunk::from_pretty( " I I I @@ -207,22 +198,11 @@ mod tests { AggCall::from_pretty("(sum:int8 $1:int8)"), ]; let schema = generate_agg_schema(&source, &agg_calls, None); - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "StatelessSimpleAggExecutor".to_string(), - }; - - let simple_agg = Box::new( - StatelessSimpleAggExecutor::new( - ActorContext::for_test(123), - info, - Box::new(source), - agg_calls, - ) - .unwrap(), - ); - let mut simple_agg = simple_agg.execute(); + + let simple_agg = + StatelessSimpleAggExecutor::new(ActorContext::for_test(123), source, schema, agg_calls) + .unwrap(); + let mut simple_agg = simple_agg.boxed().execute(); // Consume the init barrier simple_agg.next().await.unwrap().unwrap(); diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index e76ae8c391b79..3370bf4747df4 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -21,7 +21,7 @@ use tokio::sync::mpsc; use super::error::StreamExecutorError; use super::{ - Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, MessageStream, PkIndices, + Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, MessageStream, StreamChunk, StreamExecutorResult, Watermark, }; @@ -44,7 +44,6 @@ pub mod prelude { } pub struct MockSource { - info: ExecutorInfo, rx: mpsc::UnboundedReceiver, /// Whether to send a `Stop` barrier on stream finish. @@ -104,51 +103,39 @@ impl MessageSender { } } +// TODO() impl std::fmt::Debug for MockSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("MockSource") - .field("schema", &self.info.schema) - .field("pk_indices", &self.info.pk_indices) + // .field("schema", &self.info.schema) + // .field("pk_indices", &self.info.pk_indices) .finish() } } impl MockSource { - fn new( - schema: Schema, - pk_indices: PkIndices, - rx: mpsc::UnboundedReceiver, - stop_on_finish: bool, - ) -> Self { - Self { - info: ExecutorInfo { - schema, - pk_indices, - identity: "MockSource".to_string(), - }, - rx, - stop_on_finish, - } + fn new(rx: mpsc::UnboundedReceiver, stop_on_finish: bool) -> Self { + Self { rx, stop_on_finish } } #[allow(dead_code)] - pub fn channel(schema: Schema, pk_indices: PkIndices) -> (MessageSender, Self) { + pub fn channel() -> (MessageSender, Self) { let (tx, rx) = mpsc::unbounded_channel(); - let source = Self::new(schema, pk_indices, rx, true); + let source = Self::new(rx, true); (MessageSender(tx), source) } #[allow(dead_code)] - pub fn with_messages(schema: Schema, pk_indices: PkIndices, msgs: Vec) -> Self { - let (tx, source) = Self::channel(schema, pk_indices); + pub fn with_messages(msgs: Vec) -> Self { + let (tx, source) = Self::channel(); for msg in msgs { tx.0.send(msg).unwrap(); } source } - pub fn with_chunks(schema: Schema, pk_indices: PkIndices, chunks: Vec) -> Self { - let (tx, source) = Self::channel(schema, pk_indices); + pub fn with_chunks(chunks: Vec) -> Self { + let (tx, source) = Self::channel(); for chunk in chunks { tx.0.send(Message::Chunk(chunk)).unwrap(); } @@ -164,6 +151,17 @@ impl MockSource { } } + pub fn to_executor(self, schema: Schema, pk_indices: Vec) -> Executor { + Executor::new( + ExecutorInfo { + schema, + pk_indices, + identity: "MockSource".to_string(), + }, + self.boxed(), + ) + } + #[try_stream(ok = Message, error = StreamExecutorError)] async fn execute_inner(mut self: Box) { let mut epoch = 1; @@ -294,7 +292,7 @@ pub mod agg_executor { }; use crate::executor::aggregation::AggStateStorage; use crate::executor::{ - ActorContext, ActorContextRef, Execute, Executor, ExecutorInfo, HashAggExecutor, PkIndices, + ActorContext, ActorContextRef, Executor, ExecutorInfo, HashAggExecutor, PkIndices, SimpleAggExecutor, }; diff --git a/src/stream/src/executor/top_n/group_top_n.rs b/src/stream/src/executor/top_n/group_top_n.rs index 11e29d7ddfc72..e99f7e8392fd9 100644 --- a/src/stream/src/executor/top_n/group_top_n.rs +++ b/src/stream/src/executor/top_n/group_top_n.rs @@ -285,7 +285,7 @@ mod tests { use super::*; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Barrier, Message}; + use crate::executor::{ActorContext, Barrier, Execute, Message}; fn create_schema() -> Schema { Schema { @@ -351,24 +351,21 @@ mod tests { vec![chunk0, chunk1, chunk2, chunk3] } - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = create_stream_chunks(); let schema = create_schema(); - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Barrier(Barrier::new_test_barrier(4)), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(5)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Barrier(Barrier::new_test_barrier(4)), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(5)), + ]) + .to_executor(schema, pk_indices()) } #[tokio::test] @@ -384,26 +381,20 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), // this includes group key as prefix - identity: "GroupTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - GroupTopNExecutor::::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (0, 2), - order_by_1(), - vec![1], - state_table, - Arc::new(AtomicU64::new(0)), - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = GroupTopNExecutor::::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 2), + order_by_1(), + vec![1], + state_table, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -486,26 +477,20 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), // this includes group key as prefix - identity: "GroupTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - GroupTopNExecutor::::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (1, 2), - order_by_1(), - vec![1], - state_table, - Arc::new(AtomicU64::new(0)), - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = GroupTopNExecutor::::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 2), + order_by_1(), + vec![1], + state_table, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -581,26 +566,20 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), // this includes group key as prefix - identity: "GroupTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - GroupTopNExecutor::::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (0, 2), - order_by_2(), - vec![1, 2], - state_table, - Arc::new(AtomicU64::new(0)), - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = GroupTopNExecutor::::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 2), + order_by_2(), + vec![1, 2], + state_table, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); diff --git a/src/stream/src/executor/top_n/top_n_appendonly.rs b/src/stream/src/executor/top_n/top_n_appendonly.rs index f1a477344c90a..5d0b32ae149cb 100644 --- a/src/stream/src/executor/top_n/top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/top_n_appendonly.rs @@ -25,7 +25,7 @@ use super::{ManagedTopNState, TopNCache, NO_GROUP_KEY}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Execute, Executor, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need /// to keep all the rows seen. As long as a record @@ -167,7 +167,7 @@ mod tests { use super::AppendOnlyTopNExecutor; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Barrier, Execute, ExecutorInfo, Message, PkIndices}; + use crate::executor::{ActorContext, Barrier, Execute, Executor, Message, PkIndices}; fn create_stream_chunks() -> Vec { let chunk1 = StreamChunk::from_pretty( @@ -220,21 +220,17 @@ mod tests { vec![0, 1] } - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = create_stream_chunks(); - let schema = create_schema(); - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(std::mem::take(&mut chunks[2])), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(std::mem::take(&mut chunks[2])), + ]) + .to_executor(create_schema(), pk_indices()) } #[tokio::test] @@ -248,24 +244,18 @@ mod tests { ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "AppendOnlyTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - AppendOnlyTopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key, - (0, 5), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = AppendOnlyTopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key, + (0, 5), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init epoch top_n_executor.next().await.unwrap().unwrap(); @@ -335,24 +325,18 @@ mod tests { ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "AppendOnlyTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - AppendOnlyTopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (3, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = AppendOnlyTopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (3, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init epoch top_n_executor.next().await.unwrap().unwrap(); diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs index 7ff65ea64183d..ba034866c52ed 100644 --- a/src/stream/src/executor/top_n/top_n_plain.rs +++ b/src/stream/src/executor/top_n/top_n_plain.rs @@ -24,7 +24,7 @@ use super::{ManagedTopNState, TopNCache, TopNCacheTrait}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Execute, Executor, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; /// `TopNExecutor` works with input with modification, it keeps all the data /// records/rows that have been seen, and returns topN records overall. @@ -62,16 +62,16 @@ impl TopNExecutor { #[allow(clippy::too_many_arguments)] #[cfg(test)] pub fn new_with_ties_for_test( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, state_table: StateTable, ) -> StreamResult { let mut inner = - InnerTopNExecutor::new(info, storage_key, offset_and_limit, order_by, state_table)?; + InnerTopNExecutor::new(schema, storage_key, offset_and_limit, order_by, state_table)?; inner.cache.high_capacity = 2; @@ -203,7 +203,7 @@ mod tests { mod test1 { use super::*; - use crate::executor::ActorContext; + use crate::executor::{ActorContext, Execute}; fn create_stream_chunks() -> Vec { let chunk1 = StreamChunk::from_pretty( " I I @@ -262,24 +262,21 @@ mod tests { vec![0, 1] } - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = create_stream_chunks(); let schema = create_schema(); - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Barrier(Barrier::new_test_barrier(2)), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Barrier(Barrier::new_test_barrier(4)), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(5)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Barrier(Barrier::new_test_barrier(2)), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Barrier(Barrier::new_test_barrier(4)), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(5)), + ]) + .to_executor(schema, pk_indices()) } #[tokio::test] @@ -292,24 +289,18 @@ mod tests { ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (3, 1000), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (3, 1000), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -393,24 +384,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (0, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -506,24 +491,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, true>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (0, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, true>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -618,24 +597,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (3, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (3, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -712,9 +685,9 @@ mod tests { use super::*; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table_from_state_store; - use crate::executor::ActorContext; + use crate::executor::{ActorContext, Execute}; - fn create_source_new() -> Box { + fn create_source_new() -> Executor { let mut chunks = vec![ StreamChunk::from_pretty( " I I I I @@ -743,21 +716,18 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(2)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(2)), + ]) + .to_executor(schema, pk_indices()) } - fn create_source_new_before_recovery() -> Box { + fn create_source_new_before_recovery() -> Executor { let mut chunks = [ StreamChunk::from_pretty( " I I I I @@ -776,19 +746,16 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(2)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(2)), + ]) + .to_executor(schema, pk_indices()) } - fn create_source_new_after_recovery() -> Box { + fn create_source_new_after_recovery() -> Executor { let mut chunks = [ StreamChunk::from_pretty( " I I I I @@ -809,16 +776,13 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(4)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(4)), + ]) + .to_executor(schema, pk_indices()) } fn storage_key() -> Vec { @@ -850,24 +814,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (1, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -934,24 +892,18 @@ mod tests { ) .await; let source = create_source_new_before_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (1, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -994,24 +946,18 @@ mod tests { // recovery let source = create_source_new_after_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor_after_recovery = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (1, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor_after_recovery.execute(); + let schema = source.schema().clone(); + let top_n_executor_after_recovery = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor_after_recovery.boxed().execute(); // barrier assert_matches!( @@ -1054,9 +1000,9 @@ mod tests { use super::*; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table_from_state_store; - use crate::executor::ActorContext; + use crate::executor::{ActorContext, Execute}; - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = vec![ StreamChunk::from_pretty( " I I @@ -1091,18 +1037,15 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(2)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(2)), + ]) + .to_executor(schema, pk_indices()) } fn storage_key() -> Vec { @@ -1128,24 +1071,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::new_with_ties_for_test( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (0, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::new_with_ties_for_test( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -1205,7 +1142,7 @@ mod tests { ); } - fn create_source_before_recovery() -> Box { + fn create_source_before_recovery() -> Executor { let mut chunks = [ StreamChunk::from_pretty( " I I @@ -1232,19 +1169,16 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(1)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(2)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(1)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(2)), + ]) + .to_executor(schema, pk_indices()) } - fn create_source_after_recovery() -> Box { + fn create_source_after_recovery() -> Executor { let mut chunks = [ StreamChunk::from_pretty( " I I @@ -1261,16 +1195,13 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(3)), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(4)), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(3)), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(4)), + ]) + .to_executor(schema, pk_indices()) } #[tokio::test] @@ -1284,24 +1215,18 @@ mod tests { ) .await; let source = create_source_before_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::new_with_ties_for_test( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (0, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::new_with_ties_for_test( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -1347,24 +1272,18 @@ mod tests { // recovery let source = create_source_after_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor_after_recovery = Box::new( - TopNExecutor::new_with_ties_for_test( - source as Box, - ActorContext::for_test(0), - info, - storage_key(), - (0, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor_after_recovery.execute(); + let schema = source.schema().clone(); + let top_n_executor_after_recovery = TopNExecutor::new_with_ties_for_test( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor_after_recovery.boxed().execute(); // barrier assert_matches!( diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index 52ed98490f1bc..d7deee71f1443 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -152,7 +152,7 @@ mod tests { use super::ValuesExecutor; use crate::executor::test_utils::StreamExecutorTestExt; - use crate::executor::{ActorContext, AddMutation, Barrier, Execute, ExecutorInfo, Mutation}; + use crate::executor::{ActorContext, AddMutation, Barrier, Execute, Mutation}; use crate::task::{CreateMviewProgress, LocalBarrierManager}; #[tokio::test] @@ -191,15 +191,9 @@ mod tests { .iter() // for each column .map(|col| Field::unnamed(col.return_type())) .collect::(); - let pk_indices = vec![schema.len() - 1]; - let info = ExecutorInfo { - schema, - pk_indices, - identity: "ValuesExecutor".to_string(), - }; let values_executor_struct = ValuesExecutor::new( ActorContext::for_test(actor_id), - info, + schema, progress, vec![exprs .into_iter() diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 96dcc512e62d0..3954a5c0e525e 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -459,7 +459,8 @@ mod tests { ) .await; - let (tx, source) = MockSource::channel(schema, vec![0]); + let (tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![0]); let info = ExecutorInfo { schema: source.schema().clone(), @@ -470,8 +471,8 @@ mod tests { ( WatermarkFilterExecutor::new( ActorContext::for_test(123), - info, - source.boxed(), + &info, + source, watermark_expr, 1, table, diff --git a/src/stream/src/executor/wrapper/epoch_check.rs b/src/stream/src/executor/wrapper/epoch_check.rs index ab31420011397..11e5e1a704ad0 100644 --- a/src/stream/src/executor/wrapper/epoch_check.rs +++ b/src/stream/src/executor/wrapper/epoch_check.rs @@ -80,18 +80,18 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Execute; #[tokio::test] async fn test_epoch_ok() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(Default::default(), vec![]); tx.push_barrier(1, false); tx.push_chunk(StreamChunk::default()); tx.push_barrier(2, false); tx.push_barrier(3, false); tx.push_barrier(4, false); - let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == 1); @@ -104,14 +104,15 @@ mod tests { #[should_panic] #[tokio::test] async fn test_epoch_bad() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(Default::default(), vec![]); tx.push_barrier(100, false); tx.push_chunk(StreamChunk::default()); tx.push_barrier(514, false); tx.push_barrier(514, false); tx.push_barrier(114, false); - let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == 100); @@ -125,11 +126,12 @@ mod tests { #[should_panic] #[tokio::test] async fn test_epoch_first_not_barrier() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); tx.push_barrier(114, false); - let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -137,9 +139,11 @@ mod tests { #[tokio::test] async fn test_empty() { - let (_, mut source) = MockSource::channel(Default::default(), vec![]); - source = source.stop_on_finish(false); - let checked = epoch_check(source.info().clone().into(), source.boxed().execute()); + let (_, mut source) = MockSource::channel(); + let source = source + .stop_on_finish(false) + .to_executor(Default::default(), vec![]); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert!(checked.next().await.transpose().unwrap().is_none()); diff --git a/src/stream/src/executor/wrapper/schema_check.rs b/src/stream/src/executor/wrapper/schema_check.rs index 17ae1baca11df..0092fde46bf20 100644 --- a/src/stream/src/executor/wrapper/schema_check.rs +++ b/src/stream/src/executor/wrapper/schema_check.rs @@ -62,7 +62,6 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Execute; #[tokio::test] async fn test_schema_ok() { @@ -73,7 +72,8 @@ mod tests { ], }; - let (mut tx, source) = MockSource::channel(schema, vec![1]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![1]); tx.push_chunk(StreamChunk::from_pretty( " I F + 100 200.0 @@ -82,7 +82,7 @@ mod tests { )); tx.push_barrier(1, false); - let checked = schema_check(source.info().clone().into(), source.boxed().execute()); + let checked = schema_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Chunk(_)); @@ -99,7 +99,8 @@ mod tests { ], }; - let (mut tx, source) = MockSource::channel(schema, vec![1]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![1]); tx.push_chunk(StreamChunk::from_pretty( " I I + 100 200 @@ -108,7 +109,7 @@ mod tests { )); tx.push_barrier(1, false); - let checked = schema_check(source.info().clone().into(), source.boxed().execute()); + let checked = schema_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); } diff --git a/src/stream/src/executor/wrapper/update_check.rs b/src/stream/src/executor/wrapper/update_check.rs index ae98e265285ff..cb00707adafa5 100644 --- a/src/stream/src/executor/wrapper/update_check.rs +++ b/src/stream/src/executor/wrapper/update_check.rs @@ -62,12 +62,12 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Execute; #[should_panic] #[tokio::test] async fn test_not_next_to_each_other() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U- 114 @@ -76,7 +76,7 @@ mod tests { U+ 810", )); - let checked = update_check(source.info().clone().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -85,13 +85,14 @@ mod tests { #[should_panic] #[tokio::test] async fn test_first_one_update_insert() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U+ 114", )); - let checked = update_check(source.info().clone().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -100,7 +101,8 @@ mod tests { #[should_panic] #[tokio::test] async fn test_last_one_update_delete() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U- 114 @@ -108,7 +110,7 @@ mod tests { U- 1919810", )); - let checked = update_check(source.info().clone().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -116,10 +118,11 @@ mod tests { #[tokio::test] async fn test_empty_chunk() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); - let checked = update_check(source.info().clone().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); diff --git a/src/stream/src/from_proto/now.rs b/src/stream/src/from_proto/now.rs index ccfff43f3cf49..06de0cacb7197 100644 --- a/src/stream/src/from_proto/now.rs +++ b/src/stream/src/from_proto/now.rs @@ -19,7 +19,7 @@ use tokio::sync::mpsc::unbounded_channel; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{Execute, Executor, NowExecutor}; +use crate::executor::{Executor, NowExecutor}; use crate::task::ExecutorParams; pub struct NowExecutorBuilder; diff --git a/src/stream/tests/integration_tests/eowc_over_window.rs b/src/stream/tests/integration_tests/eowc_over_window.rs index 6941d86f3a759..f97fd4dbf779f 100644 --- a/src/stream/tests/integration_tests/eowc_over_window.rs +++ b/src/stream/tests/integration_tests/eowc_over_window.rs @@ -14,9 +14,7 @@ use risingwave_expr::aggregate::{AggArgs, AggKind}; use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncCall, WindowFuncKind}; -use risingwave_stream::executor::{ - EowcOverWindowExecutor, EowcOverWindowExecutorArgs, ExecutorInfo, -}; +use risingwave_stream::executor::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs}; use crate::prelude::*; @@ -54,7 +52,6 @@ async fn create_executor( }); Schema { fields } }; - let output_pk_indices = vec![2]; let state_table = StateTable::new_without_distribution_inconsistent_op( store, @@ -65,17 +62,14 @@ async fn create_executor( ) .await; - let (tx, source) = MockSource::channel(input_schema, input_pk_indices.clone()); + let (tx, source) = MockSource::channel(); + let source = source.to_executor(input_schema, input_pk_indices.clone()); let executor = EowcOverWindowExecutor::new(EowcOverWindowExecutorArgs { actor_ctx: ActorContext::for_test(123), - info: ExecutorInfo { - schema: output_schema, - pk_indices: output_pk_indices, - identity: "EowcOverWindowExecutor".to_string(), - }, - input: source.boxed(), + input: source, + schema: output_schema, calls, partition_key_indices, order_key_index, diff --git a/src/stream/tests/integration_tests/hash_agg.rs b/src/stream/tests/integration_tests/hash_agg.rs index bf82a2986bf7d..36fda81f1e209 100644 --- a/src/stream/tests/integration_tests/hash_agg.rs +++ b/src/stream/tests/integration_tests/hash_agg.rs @@ -38,10 +38,11 @@ async fn test_hash_agg_count_sum() { AggCall::from_pretty("(sum:int8 $2:int8)"), ]; - let (mut tx, source) = MockSource::channel(schema, PkIndices::new()); + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, PkIndices::new()); let hash_agg = new_boxed_hash_agg_executor( store, - Box::new(source), + source, false, agg_calls, 0, @@ -116,10 +117,11 @@ async fn test_hash_agg_min() { AggCall::from_pretty("(min:int8 $1:int8)"), ]; - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![2]); let hash_agg = new_boxed_hash_agg_executor( store, - Box::new(source), + source, false, agg_calls, 0, @@ -191,10 +193,11 @@ async fn test_hash_agg_min_append_only() { AggCall::from_pretty("(min:int8 $1:int8)"), ]; - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk + let (mut tx, source) = MockSource::channel(); + let source = source.to_executor(schema, vec![2]); let hash_agg = new_boxed_hash_agg_executor( store, - Box::new(source), + source, true, // is append only agg_calls, 0, @@ -266,10 +269,11 @@ async fn test_hash_agg_emit_on_window_close() { let agg_calls = vec![AggCall::from_pretty("(count:int8)")]; let create_executor = || async { - let (tx, source) = MockSource::channel(input_schema.clone(), PkIndices::new()); + let (tx, source) = MockSource::channel(); + let source = source.to_executor(input_schema.clone(), PkIndices::new()); let hash_agg = new_boxed_hash_agg_executor( store.clone(), - Box::new(source), + source, false, agg_calls.clone(), 0, diff --git a/src/stream/tests/integration_tests/hop_window.rs b/src/stream/tests/integration_tests/hop_window.rs index f0bd65c84b69e..a0502f2744c22 100644 --- a/src/stream/tests/integration_tests/hop_window.rs +++ b/src/stream/tests/integration_tests/hop_window.rs @@ -16,7 +16,7 @@ use risingwave_common::types::test_utils::IntervalTestExt; use risingwave_common::types::{Interval, Timestamp}; use risingwave_expr::expr::test_utils::make_hop_window_expression; use risingwave_expr::expr::NonStrictExpression; -use risingwave_stream::executor::{ExecutorInfo, HopWindowExecutor}; +use risingwave_stream::executor::HopWindowExecutor; use crate::prelude::*; @@ -24,12 +24,15 @@ const TIME_COL_IDX: usize = 2; const CHUNK_SIZE: usize = 256; fn create_executor(output_indices: Vec) -> (MessageSender, BoxedMessageStream) { - let field1 = Field::unnamed(DataType::Int64); - let field2 = Field::unnamed(DataType::Int64); - let field3 = Field::with_name(DataType::Timestamp, "created_at"); - let schema = Schema::new(vec![field1, field2, field3]); - let pk_indices = vec![0]; - let (tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); + let (tx, source) = MockSource::channel(); + let source = source.to_executor( + Schema::new(vec![ + Field::unnamed(DataType::Int64), + Field::unnamed(DataType::Int64), + Field::with_name(DataType::Timestamp, "created_at"), + ]), + vec![0], + ); let window_slide = Interval::from_minutes(15); let window_size = Interval::from_minutes(30); @@ -47,12 +50,7 @@ fn create_executor(output_indices: Vec) -> (MessageSender, BoxedMessageSt tx, HopWindowExecutor::new( ActorContext::for_test(123), - ExecutorInfo { - schema, - pk_indices, - identity: "HopWindowExecutor".to_string(), - }, - Box::new(source), + source, TIME_COL_IDX, window_slide, window_size, diff --git a/src/stream/tests/integration_tests/over_window.rs b/src/stream/tests/integration_tests/over_window.rs index 78203d79a87ac..2d7d8a8a13f8e 100644 --- a/src/stream/tests/integration_tests/over_window.rs +++ b/src/stream/tests/integration_tests/over_window.rs @@ -18,7 +18,7 @@ use risingwave_expr::window_function::{ Frame, FrameBound, FrameExclusion, WindowFuncCall, WindowFuncKind, }; use risingwave_stream::executor::monitor::StreamingMetrics; -use risingwave_stream::executor::{ExecutorInfo, OverWindowExecutor, OverWindowExecutorArgs}; +use risingwave_stream::executor::{OverWindowExecutor, OverWindowExecutorArgs}; use crate::prelude::*; @@ -63,7 +63,6 @@ async fn create_executor( }); Schema { fields } }; - let output_pk_indices = vec![2]; let state_table = StateTable::new_without_distribution( store, @@ -74,17 +73,14 @@ async fn create_executor( ) .await; - let (tx, source) = MockSource::channel(input_schema, input_pk_indices.clone()); + let (tx, source) = MockSource::channel(); + let source = source.to_executor(input_schema, input_pk_indices.clone()); let executor = OverWindowExecutor::new(OverWindowExecutorArgs { actor_ctx: ActorContext::for_test(123), - info: ExecutorInfo { - schema: output_schema, - pk_indices: output_pk_indices, - identity: "OverWindowExecutor".to_string(), - }, - input: source.boxed(), + input: source, + schema: output_schema, calls, partition_key_indices, order_key_indices, diff --git a/src/stream/tests/integration_tests/project_set.rs b/src/stream/tests/integration_tests/project_set.rs index 79fcbfc0d48db..be8983cdfaa09 100644 --- a/src/stream/tests/integration_tests/project_set.rs +++ b/src/stream/tests/integration_tests/project_set.rs @@ -14,7 +14,7 @@ use multimap::MultiMap; use risingwave_expr::table_function::repeat; -use risingwave_stream::executor::{ExecutorInfo, ProjectSetExecutor}; +use risingwave_stream::executor::ProjectSetExecutor; use crate::prelude::*; @@ -27,30 +27,17 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) { Field::unnamed(DataType::Int64), ], }; - let (tx, source) = MockSource::channel(schema, PkIndices::new()); + let (tx, source) = MockSource::channel(); + let source = source.to_executor(schema, PkIndices::new()); let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)").into_inner(); let test_expr_watermark = build_from_pretty("(add:int8 $0:int8 1:int8)").into_inner(); let tf1 = repeat(build_from_pretty("1:int4").into_inner(), 1); let tf2 = repeat(build_from_pretty("2:int4").into_inner(), 2); - let info = ExecutorInfo { - schema: Schema { - fields: vec![ - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int32), - Field::unnamed(DataType::Int32), - ], - }, - pk_indices: vec![], - identity: "ProjectSetExecutor".to_string(), - }; - - let project_set = Box::new(ProjectSetExecutor::new( + let project_set = ProjectSetExecutor::new( ActorContext::for_test(123), - info, - Box::new(source), + source, vec![ test_expr.into(), test_expr_watermark.into(), @@ -60,8 +47,8 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) { CHUNK_SIZE, MultiMap::from_iter(std::iter::once((0, 1))), vec![], - )); - (tx, project_set.execute()) + ); + (tx, project_set.boxed().execute()) } #[tokio::test] From a78bbde28adff2de1a42e6e94a43c641366704ed Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Thu, 22 Feb 2024 23:47:01 +0800 Subject: [PATCH 10/13] fix clippy Signed-off-by: Richard Chien --- src/compute/tests/cdc_tests.rs | 2 +- src/stream/benches/stream_hash_agg.rs | 2 +- .../executor/backfill/arrangement_backfill.rs | 4 ++-- .../src/executor/backfill/cdc/cdc_backfill.rs | 3 +-- .../executor/backfill/no_shuffle_backfill.rs | 4 ++-- src/stream/src/executor/barrier_recv.rs | 5 ++-- src/stream/src/executor/chain.rs | 4 ++-- .../src/executor/dedup/append_only_dedup.rs | 2 +- src/stream/src/executor/dml.rs | 2 +- src/stream/src/executor/dynamic_filter.rs | 4 ++-- src/stream/src/executor/expand.rs | 2 +- src/stream/src/executor/filter.rs | 7 +----- src/stream/src/executor/hash_join.rs | 8 +++---- src/stream/src/executor/hop_window.rs | 2 +- src/stream/src/executor/lookup.rs | 4 +--- src/stream/src/executor/lookup/impl_.rs | 1 - src/stream/src/executor/lookup/sides.rs | 8 ++++--- src/stream/src/executor/lookup/tests.rs | 8 +++---- src/stream/src/executor/lookup_union.rs | 6 ++--- src/stream/src/executor/merge.rs | 2 -- src/stream/src/executor/mview/materialize.rs | 16 ++++++------- src/stream/src/executor/over_window/eowc.rs | 4 ++-- src/stream/src/executor/project.rs | 24 ++----------------- src/stream/src/executor/row_id_gen.rs | 2 +- src/stream/src/executor/simple_agg.rs | 2 +- src/stream/src/executor/sink.rs | 6 ++--- src/stream/src/executor/sort.rs | 2 +- .../src/executor/source/source_executor.rs | 2 -- .../src/executor/stateless_simple_agg.rs | 4 ++-- src/stream/src/executor/subscription.rs | 4 ++-- src/stream/src/executor/subtask.rs | 12 ++-------- src/stream/src/executor/test_utils.rs | 2 +- src/stream/src/executor/top_n/group_top_n.rs | 2 +- .../src/executor/top_n/top_n_appendonly.rs | 2 +- src/stream/src/executor/top_n/top_n_plain.rs | 14 +++++------ src/stream/src/executor/top_n/utils.rs | 4 ++-- src/stream/src/executor/utils.rs | 2 +- src/stream/src/executor/watermark_filter.rs | 2 +- .../src/executor/wrapper/epoch_check.rs | 10 ++++---- .../src/executor/wrapper/schema_check.rs | 4 ++-- .../src/executor/wrapper/update_check.rs | 8 +++---- src/stream/src/from_proto/eowc_over_window.rs | 2 +- src/stream/src/from_proto/over_window.rs | 2 +- .../integration_tests/eowc_over_window.rs | 2 +- .../tests/integration_tests/hash_agg.rs | 8 +++---- .../tests/integration_tests/hop_window.rs | 2 +- .../tests/integration_tests/over_window.rs | 2 +- .../tests/integration_tests/project_set.rs | 2 +- 48 files changed, 93 insertions(+), 135 deletions(-) diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs index 583e35dc9b7f6..1347563922984 100644 --- a/src/compute/tests/cdc_tests.rs +++ b/src/compute/tests/cdc_tests.rs @@ -134,7 +134,7 @@ async fn test_cdc_backfill() -> StreamResult<()> { let memory_state_store = MemoryStateStore::new(); let (mut tx, source) = MockSource::channel(); - let source = source.to_executor( + let source = source.into_executor( Schema::new(vec![ Field::unnamed(DataType::Jsonb), // payload ]), diff --git a/src/stream/benches/stream_hash_agg.rs b/src/stream/benches/stream_hash_agg.rs index e10cd2629a8bc..2a0d1a93cc6b7 100644 --- a/src/stream/benches/stream_hash_agg.rs +++ b/src/stream/benches/stream_hash_agg.rs @@ -120,7 +120,7 @@ fn setup_bench_hash_agg(store: S) -> Executor { // ---- Create MockSourceExecutor ---- let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, PkIndices::new()); + let source = source.into_executor(schema, PkIndices::new()); tx.push_barrier(1, false); for chunk in chunks { tx.push_chunk(chunk); diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 5ce65448c0160..2dd277449be13 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -39,8 +39,8 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, HashMap, - Message, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, HashMap, Message, + StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index 34dc60e5088e2..e4b83622951f6 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -624,7 +624,6 @@ mod tests { use crate::executor::backfill::cdc::cdc_backfill::transform_upstream; use crate::executor::test_utils::MockSource; - use crate::executor::Execute; #[tokio::test] async fn test_transform_upstream_chunk() { @@ -635,7 +634,7 @@ mod tests { ]); let pk_indices = vec![1]; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema.clone(), pk_indices.clone()); + let source = source.into_executor(schema.clone(), pk_indices.clone()); // let payload = r#"{"before": null,"after":{"O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" },"source":{"version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null},"op":"r","ts_ms":1695277757017,"transaction":null}"#.to_string(); let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#; diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index 6b229baf7ad2a..4aea8f63b26ea 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -38,8 +38,8 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, - Mutation, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, Message, Mutation, + StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; diff --git a/src/stream/src/executor/barrier_recv.rs b/src/stream/src/executor/barrier_recv.rs index a230c5fa952c7..4ca07a80382a5 100644 --- a/src/stream/src/executor/barrier_recv.rs +++ b/src/stream/src/executor/barrier_recv.rs @@ -13,13 +13,12 @@ // limitations under the License. use futures::StreamExt; -use risingwave_common::catalog::Schema; use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ - ActorContext, ActorContextRef, Barrier, BoxedMessageStream, Execute, ExecutorInfo, Message, - PkIndices, StreamExecutorError, + ActorContext, ActorContextRef, Barrier, BoxedMessageStream, Execute, Message, + StreamExecutorError, }; /// The executor only for receiving barrier from the meta service. It always resides in the leaves diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index fdd1daff02e87..3e65bf878b552 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -133,7 +133,7 @@ mod test { StreamChunk::from_pretty("I\n + 2"), ]) .stop_on_finish(false) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let second = MockSource::with_messages(vec![ Message::Barrier(Barrier::new_test_barrier(1).with_mutation(Mutation::Add( @@ -152,7 +152,7 @@ mod test { Message::Chunk(StreamChunk::from_pretty("I\n + 3")), Message::Chunk(StreamChunk::from_pretty("I\n + 4")), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let chain = ChainExecutor::new(first, second, progress, false); diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index e1e711207ff72..34b102ff1e030 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -232,7 +232,7 @@ mod tests { .await; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, pk_indices); + let source = source.into_executor(schema, pk_indices); let mut dedup_executor = AppendOnlyDedupExecutor::new( ActorContext::for_test(123), source, diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index e25340f811aab..0463c695639d7 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -307,7 +307,7 @@ mod tests { let dml_manager = Arc::new(DmlManager::for_test()); let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, pk_indices); + let source = source.into_executor(schema, pk_indices); let dml_executor = DmlExecutor::new( source, diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index 6c17c380c02fe..6bcd3e97ed563 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -551,9 +551,9 @@ mod tests { fields: vec![Field::unnamed(DataType::Int64)], }; let (tx_l, source_l) = MockSource::channel(); - let source_l = source_l.to_executor(schema.clone(), vec![0]); + let source_l = source_l.into_executor(schema.clone(), vec![0]); let (tx_r, source_r) = MockSource::channel(); - let source_r = source_r.to_executor(schema, vec![]); + let source_r = source_r.into_executor(schema, vec![]); let executor = DynamicFilterExecutor::::new( ActorContext::for_test(123), diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index 5e5672b035bce..2375ebca3d0df 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -92,7 +92,7 @@ mod tests { + 6 6 3 - 7 5 4", ); - let source = MockSource::with_chunks(vec![chunk1]).to_executor( + let source = MockSource::with_chunks(vec![chunk1]).into_executor( Schema::new(vec![ Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index a3b908ec53797..ffc8847c42abf 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -204,12 +204,7 @@ mod tests { }; let pk_indices = PkIndices::new(); let source = MockSource::with_chunks(vec![chunk1, chunk2]) - .to_executor(schema.clone(), pk_indices.clone()); - let info = ExecutorInfo { - schema, - pk_indices, - identity: "FilterExecutor".to_string(), - }; + .into_executor(schema.clone(), pk_indices.clone()); let test_expr = build_from_pretty("(greater_than:boolean $0:int8 $1:int8)"); diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index 3c3728d3eb428..3727da397d01f 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -1143,9 +1143,9 @@ mod tests { ], }; let (tx_l, source_l) = MockSource::channel(); - let source_l = source_l.to_executor(schema.clone(), vec![1]); + let source_l = source_l.into_executor(schema.clone(), vec![1]); let (tx_r, source_r) = MockSource::channel(); - let source_r = source_r.to_executor(schema, vec![1]); + let source_r = source_r.into_executor(schema, vec![1]); let params_l = JoinParams::new(vec![0], vec![1]); let params_r = JoinParams::new(vec![0], vec![1]); let cond = with_condition.then(|| create_cond(condition_text)); @@ -1227,9 +1227,9 @@ mod tests { ], }; let (tx_l, source_l) = MockSource::channel(); - let source_l = source_l.to_executor(schema.clone(), vec![0]); + let source_l = source_l.into_executor(schema.clone(), vec![0]); let (tx_r, source_r) = MockSource::channel(); - let source_r = source_r.to_executor(schema, vec![0]); + let source_r = source_r.into_executor(schema, vec![0]); let params_l = JoinParams::new(vec![0, 1], vec![]); let params_r = JoinParams::new(vec![0, 1], vec![]); let cond = with_condition.then(|| create_cond(None)); diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs index a839d0075f372..801f3daa9f53a 100644 --- a/src/stream/src/executor/hop_window.rs +++ b/src/stream/src/executor/hop_window.rs @@ -264,7 +264,7 @@ mod tests { .replace('^', "2022-02-02T"), ); let input = - MockSource::with_chunks(vec![chunk]).to_executor(schema.clone(), pk_indices.clone()); + MockSource::with_chunks(vec![chunk]).into_executor(schema.clone(), pk_indices.clone()); let window_slide = Interval::from_minutes(15); let window_size = Interval::from_minutes(30); let window_offset = Interval::from_minutes(0); diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs index bf591f2a28f00..2c1de3170a801 100644 --- a/src/stream/src/executor/lookup.rs +++ b/src/stream/src/executor/lookup.rs @@ -27,7 +27,7 @@ mod impl_; pub use impl_::LookupExecutorParams; -use super::{ActorContextRef, Executor, ExecutorInfo}; +use super::{ActorContextRef, Executor}; #[cfg(test)] mod tests; @@ -41,8 +41,6 @@ mod tests; pub struct LookupExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - /// the data types of the produced data chunk inside lookup (before reordering) chunk_data_types: Vec, diff --git a/src/stream/src/executor/lookup/impl_.rs b/src/stream/src/executor/lookup/impl_.rs index d7878abc03022..db2511e19e7bd 100644 --- a/src/stream/src/executor/lookup/impl_.rs +++ b/src/stream/src/executor/lookup/impl_.rs @@ -198,7 +198,6 @@ impl LookupExecutor { Self { ctx, - info, chunk_data_types, last_barrier: None, stream_executor: Some(stream), diff --git a/src/stream/src/executor/lookup/sides.rs b/src/stream/src/executor/lookup/sides.rs index 2c41eae40bd47..6e64519503a93 100644 --- a/src/stream/src/executor/lookup/sides.rs +++ b/src/stream/src/executor/lookup/sides.rs @@ -28,7 +28,7 @@ use risingwave_storage::table::batch_table::storage_table::StorageTable; use risingwave_storage::StateStore; use crate::executor::error::StreamExecutorError; -use crate::executor::{Barrier, BoxedMessageStream, Execute, Executor, Message, MessageStream}; +use crate::executor::{Barrier, BoxedMessageStream, Executor, Message, MessageStream}; /// Join side of Lookup Executor's stream pub(crate) struct StreamJoinSide { @@ -429,9 +429,11 @@ mod tests { let (mut tx_l, source_l) = MockSource::channel(); let source_l = source_l .stop_on_finish(false) - .to_executor(schema.clone(), vec![1]); + .into_executor(schema.clone(), vec![1]); let (tx_r, source_r) = MockSource::channel(); - let source_r = source_r.stop_on_finish(false).to_executor(schema, vec![1]); + let source_r = source_r + .stop_on_finish(false) + .into_executor(schema, vec![1]); let mut stream = stream_lookup_arrange_this_epoch(source_l, source_r).boxed(); diff --git a/src/stream/src/executor/lookup/tests.rs b/src/stream/src/executor/lookup/tests.rs index 4cda86614dc12..9b335fcca6428 100644 --- a/src/stream/src/executor/lookup/tests.rs +++ b/src/stream/src/executor/lookup/tests.rs @@ -105,7 +105,7 @@ async fn create_arrangement(table_id: TableId, memory_state_store: MemoryStateSt Message::Chunk(chunk2), Message::Barrier(Barrier::new_test_barrier(4)), ]) - .to_executor(schema, vec![0]); + .into_executor(schema, vec![0]); Executor::new( ExecutorInfo { @@ -162,16 +162,14 @@ fn create_source() -> Executor { .collect_vec(), ); - let source = MockSource::with_messages(vec![ + MockSource::with_messages(vec![ Message::Barrier(Barrier::new_test_barrier(2)), Message::Chunk(chunk1), Message::Barrier(Barrier::new_test_barrier(3)), Message::Chunk(chunk2), Message::Barrier(Barrier::new_test_barrier(4)), ]) - .to_executor(schema, PkIndices::new()); - - source + .into_executor(schema, PkIndices::new()) } async fn next_msg(buffer: &mut Vec, executor: &mut BoxedMessageStream) { diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index 7722582f8194a..bd6d48d957c67 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -148,7 +148,7 @@ mod tests { Message::Barrier(Barrier::new_test_barrier(3)), ]) .stop_on_finish(false) - .to_executor(schema.clone(), vec![0]); + .into_executor(schema.clone(), vec![0]); let source1 = MockSource::with_messages(vec![ Message::Chunk(StreamChunk::from_pretty("I\n + 11")), Message::Barrier(Barrier::new_test_barrier(1)), @@ -156,7 +156,7 @@ mod tests { Message::Barrier(Barrier::new_test_barrier(2)), ]) .stop_on_finish(false) - .to_executor(schema.clone(), vec![0]); + .into_executor(schema.clone(), vec![0]); let source2 = MockSource::with_messages(vec![ Message::Chunk(StreamChunk::from_pretty("I\n + 21")), Message::Barrier(Barrier::new_test_barrier(1)), @@ -164,7 +164,7 @@ mod tests { Message::Barrier(Barrier::new_test_barrier(2)), ]) .stop_on_finish(false) - .to_executor(schema, vec![0]); + .into_executor(schema, vec![0]); let executor = LookupUnionExecutor::new(vec![source0, source1, source2], vec![2, 1, 0]) .boxed() diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index c6fd4232ea232..e5b88dd274e1e 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -533,8 +533,6 @@ mod tests { #[tokio::test] async fn test_configuration_change() { - let schema = Schema { fields: vec![] }; - let actor_id = 233; let (untouched, old, new) = (234, 235, 238); // upstream actors let ctx = Arc::new(SharedContext::for_test()); diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index 28a494d59411a..ebf68125f8c6e 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -663,7 +663,7 @@ mod tests { Message::Chunk(chunk2), Message::Barrier(Barrier::new_test_barrier(3)), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -766,7 +766,7 @@ mod tests { Message::Chunk(chunk2), Message::Barrier(Barrier::new_test_barrier(3)), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -858,7 +858,7 @@ mod tests { Message::Chunk(chunk3), Message::Barrier(Barrier::new_test_barrier(3)), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -986,7 +986,7 @@ mod tests { Message::Chunk(chunk3), Message::Barrier(Barrier::new_test_barrier(4)), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1164,7 +1164,7 @@ mod tests { Message::Chunk(chunk3), Message::Barrier(Barrier::new_test_barrier(3)), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1267,7 +1267,7 @@ mod tests { Message::Chunk(chunk1), Message::Barrier(Barrier::new_test_barrier(2)), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1385,7 +1385,7 @@ mod tests { Message::Chunk(chunk3), Message::Barrier(Barrier::new_test_barrier(4)), ]) - .to_executor(schema.clone(), PkIndices::new()); + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1581,7 +1581,7 @@ mod tests { .collect(); // Prepare stream executors. let source = - MockSource::with_messages(messages).to_executor(schema.clone(), PkIndices::new()); + MockSource::with_messages(messages).into_executor(schema.clone(), PkIndices::new()); let mut materialize_executor = MaterializeExecutor::for_test( source, diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 720b57241e40f..32f1fafe9b8db 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -39,8 +39,8 @@ use crate::cache::{new_unbounded, ManagedLruCache}; use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, - Message, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, StreamExecutorResult, }; use crate::task::AtomicU64Ref; diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index cef4f6b997350..c44d93e22639a 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -231,18 +231,10 @@ mod tests { }; let pk_indices = vec![0]; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, pk_indices); + let source = source.into_executor(schema, pk_indices); let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)"); - let info = ExecutorInfo { - schema: Schema { - fields: vec![Field::unnamed(DataType::Int64)], - }, - pk_indices: vec![], - identity: "ProjectExecutor".to_string(), - }; - let project = ProjectExecutor::new( ActorContext::for_test(123), source, @@ -319,24 +311,12 @@ mod tests { ], }; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, PkIndices::new()); + let source = source.into_executor(schema, PkIndices::new()); let a_expr = build_from_pretty("(add:int8 $0:int8 1:int8)"); let b_expr = build_from_pretty("(subtract:int8 $0:int8 1:int8)"); let c_expr = NonStrictExpression::for_test(DummyNondecreasingExpr); - let info = ExecutorInfo { - schema: Schema { - fields: vec![ - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - ], - }, - pk_indices: vec![], - identity: "ProjectExecutor".to_string(), - }; - let project = ProjectExecutor::new( ActorContext::for_test(123), source, diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index 49c7e02a1b1ad..0a0fe1ffe84e1 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -148,7 +148,7 @@ mod tests { let row_id_index = 0; let row_id_generator = Bitmap::ones(VirtualNode::COUNT); let (mut tx, upstream) = MockSource::channel(); - let upstream = upstream.to_executor(schema.clone(), pk_indices.clone()); + let upstream = upstream.into_executor(schema.clone(), pk_indices.clone()); let row_id_gen_executor = RowIdGenExecutor::new( ActorContext::for_test(233), diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index 0eedd73125aa4..a863fc0432876 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -324,7 +324,7 @@ mod tests { ], }; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![2]); + let source = source.into_executor(schema, vec![2]); tx.push_barrier(1, false); tx.push_barrier(2, false); tx.push_chunk(StreamChunk::from_pretty( diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index 54b4cfae42fc7..7928308c3f94a 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -481,7 +481,7 @@ mod test { - 5 6 7", ))), ]) - .to_executor(schema.clone(), pk_indices.clone()); + .into_executor(schema.clone(), pk_indices.clone()); let sink_param = SinkParam { sink_id: 0.into(), @@ -605,7 +605,7 @@ mod test { ))), Message::Barrier(Barrier::new_test_barrier(3)), ]) - .to_executor(schema.clone(), vec![0, 1]); + .into_executor(schema.clone(), vec![0, 1]); let sink_param = SinkParam { sink_id: 0.into(), @@ -726,7 +726,7 @@ mod test { Message::Barrier(Barrier::new_test_barrier(2)), Message::Barrier(Barrier::new_test_barrier(3)), ]) - .to_executor(schema.clone(), pk_indices.clone()); + .into_executor(schema.clone(), pk_indices.clone()); let sink_param = SinkParam { sink_id: 0.into(), diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index d9d257458306f..72474eea32977 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -191,7 +191,7 @@ mod tests { .await; let (tx, source) = MockSource::channel(); - let source = source.to_executor(input_schema, input_pk_indices); + let source = source.into_executor(input_schema, input_pk_indices); let sort_executor = SortExecutor::new(SortExecutorArgs { actor_ctx: ActorContext::for_test(123), schema: source.schema().clone(), diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index 026181fcfe3bd..1578f52e29060 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -694,7 +694,6 @@ mod tests { fields: vec![Field::with_name(DataType::Int32, "sequence_int")], }; let row_id_index = None; - let pk_indices = vec![0]; let source_info = StreamSourceInfo { row_format: PbRowFormatType::Native as i32, ..Default::default() @@ -782,7 +781,6 @@ mod tests { fields: vec![Field::with_name(DataType::Int32, "v1")], }; let row_id_index = None; - let pk_indices = vec![0_usize]; let source_info = StreamSourceInfo { row_format: PbRowFormatType::Native as i32, ..Default::default() diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index 6ffdf5d25e565..1ad25fe5c7f6c 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -143,7 +143,7 @@ mod tests { async fn test_no_chunk() { let schema = schema_test_utils::ii(); let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![2]); + let source = source.into_executor(schema, vec![2]); tx.push_barrier(1, false); tx.push_barrier(2, false); tx.push_barrier(3, false); @@ -174,7 +174,7 @@ mod tests { async fn test_local_simple_agg() { let schema = schema_test_utils::iii(); let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![2]); + let source = source.into_executor(schema, vec![2]); tx.push_barrier(1, false); tx.push_chunk(StreamChunk::from_pretty( " I I I diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs index a6e18e2adbc27..dfe352909fd8e 100644 --- a/src/stream/src/executor/subscription.rs +++ b/src/stream/src/executor/subscription.rs @@ -22,8 +22,8 @@ use risingwave_storage::store::LocalStateStore; use tokio::time::Instant; use super::{ - expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, - Message, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, StreamExecutorResult, }; use crate::common::log_store_impl::kv_log_store::ReaderTruncationOffsetType; use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter; diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index f17851696ee38..fea9644b151f3 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -20,7 +20,7 @@ use tokio::sync::mpsc::error::SendError; use tokio_stream::wrappers::ReceiverStream; use super::actor::spawn_blocking_drop_stream; -use super::{Execute, Executor, ExecutorInfo, Message, MessageStreamItem}; +use super::{Execute, Executor, Message, MessageStreamItem}; use crate::task::ActorId; /// Handle used to drive the subtask. @@ -29,8 +29,6 @@ pub type SubtaskHandle = impl Future + Send + 'static; /// The thin wrapper for subtask-wrapped executor, containing a channel to receive the messages from /// the subtask. pub struct SubtaskRxExecutor { - info: ExecutorInfo, - rx: mpsc::Receiver, } @@ -48,13 +46,7 @@ impl Execute for SubtaskRxExecutor { /// single thread. pub fn wrap(input: Executor, actor_id: ActorId) -> (SubtaskHandle, SubtaskRxExecutor) { let (tx, rx) = mpsc::channel(1); - let rx_executor = SubtaskRxExecutor { - info: ExecutorInfo { - identity: "SubtaskRxExecutor".to_owned(), - ..input.info().clone() - }, - rx, - }; + let rx_executor = SubtaskRxExecutor { rx }; let handle = async move { let mut input = input.execute(); diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index 3370bf4747df4..56ebd9faa1fe8 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -151,7 +151,7 @@ impl MockSource { } } - pub fn to_executor(self, schema: Schema, pk_indices: Vec) -> Executor { + pub fn into_executor(self, schema: Schema, pk_indices: Vec) -> Executor { Executor::new( ExecutorInfo { schema, diff --git a/src/stream/src/executor/top_n/group_top_n.rs b/src/stream/src/executor/top_n/group_top_n.rs index e99f7e8392fd9..d4c0fa029341e 100644 --- a/src/stream/src/executor/top_n/group_top_n.rs +++ b/src/stream/src/executor/top_n/group_top_n.rs @@ -365,7 +365,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[3])), Message::Barrier(Barrier::new_test_barrier(5)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } #[tokio::test] diff --git a/src/stream/src/executor/top_n/top_n_appendonly.rs b/src/stream/src/executor/top_n/top_n_appendonly.rs index 5d0b32ae149cb..095222939a6e8 100644 --- a/src/stream/src/executor/top_n/top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/top_n_appendonly.rs @@ -230,7 +230,7 @@ mod tests { Message::Barrier(Barrier::new_test_barrier(3)), Message::Chunk(std::mem::take(&mut chunks[2])), ]) - .to_executor(create_schema(), pk_indices()) + .into_executor(create_schema(), pk_indices()) } #[tokio::test] diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs index ba034866c52ed..b09f13764ee79 100644 --- a/src/stream/src/executor/top_n/top_n_plain.rs +++ b/src/stream/src/executor/top_n/top_n_plain.rs @@ -276,7 +276,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[3])), Message::Barrier(Barrier::new_test_barrier(5)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } #[tokio::test] @@ -724,7 +724,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[3])), Message::Barrier(Barrier::new_test_barrier(2)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } fn create_source_new_before_recovery() -> Executor { @@ -752,7 +752,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[1])), Message::Barrier(Barrier::new_test_barrier(2)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } fn create_source_new_after_recovery() -> Executor { @@ -782,7 +782,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[1])), Message::Barrier(Barrier::new_test_barrier(4)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } fn storage_key() -> Vec { @@ -1045,7 +1045,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[3])), Message::Barrier(Barrier::new_test_barrier(2)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } fn storage_key() -> Vec { @@ -1175,7 +1175,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[1])), Message::Barrier(Barrier::new_test_barrier(2)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } fn create_source_after_recovery() -> Executor { @@ -1201,7 +1201,7 @@ mod tests { Message::Chunk(std::mem::take(&mut chunks[1])), Message::Barrier(Barrier::new_test_barrier(4)), ]) - .to_executor(schema, pk_indices()) + .into_executor(schema, pk_indices()) } #[tokio::test] diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index 2f238af2e8995..bbd956cde2168 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -30,8 +30,8 @@ use risingwave_common::util::sort_util::ColumnOrder; use super::CacheKey; use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, - Message, Watermark, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + Watermark, }; pub trait TopNExecutorBase: Send + 'static { diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs index 550d2a95970b8..18b64633a3e12 100644 --- a/src/stream/src/executor/utils.rs +++ b/src/stream/src/executor/utils.rs @@ -16,7 +16,7 @@ use futures::StreamExt; use risingwave_common::metrics::LabelGuardedIntCounter; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{BoxedMessageStream, Execute, ExecutorInfo}; +use crate::executor::{BoxedMessageStream, Execute}; use crate::task::{ActorId, FragmentId}; #[derive(Default)] diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 3954a5c0e525e..4875d8a53c2d1 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -460,7 +460,7 @@ mod tests { .await; let (tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![0]); + let source = source.into_executor(schema, vec![0]); let info = ExecutorInfo { schema: source.schema().clone(), diff --git a/src/stream/src/executor/wrapper/epoch_check.rs b/src/stream/src/executor/wrapper/epoch_check.rs index 11e5e1a704ad0..13d755dcbc60d 100644 --- a/src/stream/src/executor/wrapper/epoch_check.rs +++ b/src/stream/src/executor/wrapper/epoch_check.rs @@ -84,7 +84,7 @@ mod tests { #[tokio::test] async fn test_epoch_ok() { let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(Default::default(), vec![]); + let source = source.into_executor(Default::default(), vec![]); tx.push_barrier(1, false); tx.push_chunk(StreamChunk::default()); tx.push_barrier(2, false); @@ -105,7 +105,7 @@ mod tests { #[tokio::test] async fn test_epoch_bad() { let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(Default::default(), vec![]); + let source = source.into_executor(Default::default(), vec![]); tx.push_barrier(100, false); tx.push_chunk(StreamChunk::default()); tx.push_barrier(514, false); @@ -127,7 +127,7 @@ mod tests { #[tokio::test] async fn test_epoch_first_not_barrier() { let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(Default::default(), vec![]); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); tx.push_barrier(114, false); @@ -139,10 +139,10 @@ mod tests { #[tokio::test] async fn test_empty() { - let (_, mut source) = MockSource::channel(); + let (_, source) = MockSource::channel(); let source = source .stop_on_finish(false) - .to_executor(Default::default(), vec![]); + .into_executor(Default::default(), vec![]); let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); diff --git a/src/stream/src/executor/wrapper/schema_check.rs b/src/stream/src/executor/wrapper/schema_check.rs index 0092fde46bf20..ad5f0acd3dfaa 100644 --- a/src/stream/src/executor/wrapper/schema_check.rs +++ b/src/stream/src/executor/wrapper/schema_check.rs @@ -73,7 +73,7 @@ mod tests { }; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![1]); + let source = source.into_executor(schema, vec![1]); tx.push_chunk(StreamChunk::from_pretty( " I F + 100 200.0 @@ -100,7 +100,7 @@ mod tests { }; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![1]); + let source = source.into_executor(schema, vec![1]); tx.push_chunk(StreamChunk::from_pretty( " I I + 100 200 diff --git a/src/stream/src/executor/wrapper/update_check.rs b/src/stream/src/executor/wrapper/update_check.rs index cb00707adafa5..37a2ca4f2a942 100644 --- a/src/stream/src/executor/wrapper/update_check.rs +++ b/src/stream/src/executor/wrapper/update_check.rs @@ -67,7 +67,7 @@ mod tests { #[tokio::test] async fn test_not_next_to_each_other() { let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(Default::default(), vec![]); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U- 114 @@ -86,7 +86,7 @@ mod tests { #[tokio::test] async fn test_first_one_update_insert() { let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(Default::default(), vec![]); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U+ 114", @@ -102,7 +102,7 @@ mod tests { #[tokio::test] async fn test_last_one_update_delete() { let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(Default::default(), vec![]); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U- 114 @@ -119,7 +119,7 @@ mod tests { #[tokio::test] async fn test_empty_chunk() { let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(Default::default(), vec![]); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); let checked = update_check(source.info().clone().into(), source.execute()); diff --git a/src/stream/src/from_proto/eowc_over_window.rs b/src/stream/src/from_proto/eowc_over_window.rs index 81cd965cdef4c..4f6e873d7bcf3 100644 --- a/src/stream/src/from_proto/eowc_over_window.rs +++ b/src/stream/src/from_proto/eowc_over_window.rs @@ -21,7 +21,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Execute, Executor}; +use crate::executor::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Executor}; use crate::task::ExecutorParams; pub struct EowcOverWindowExecutorBuilder; diff --git a/src/stream/src/from_proto/over_window.rs b/src/stream/src/from_proto/over_window.rs index b75d2d66fd646..f7ca73c183a81 100644 --- a/src/stream/src/from_proto/over_window.rs +++ b/src/stream/src/from_proto/over_window.rs @@ -23,7 +23,7 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{Execute, Executor, OverWindowExecutor, OverWindowExecutorArgs}; +use crate::executor::{Executor, OverWindowExecutor, OverWindowExecutorArgs}; use crate::task::ExecutorParams; pub struct OverWindowExecutorBuilder; diff --git a/src/stream/tests/integration_tests/eowc_over_window.rs b/src/stream/tests/integration_tests/eowc_over_window.rs index f97fd4dbf779f..cc674e556ab5e 100644 --- a/src/stream/tests/integration_tests/eowc_over_window.rs +++ b/src/stream/tests/integration_tests/eowc_over_window.rs @@ -63,7 +63,7 @@ async fn create_executor( .await; let (tx, source) = MockSource::channel(); - let source = source.to_executor(input_schema, input_pk_indices.clone()); + let source = source.into_executor(input_schema, input_pk_indices.clone()); let executor = EowcOverWindowExecutor::new(EowcOverWindowExecutorArgs { actor_ctx: ActorContext::for_test(123), diff --git a/src/stream/tests/integration_tests/hash_agg.rs b/src/stream/tests/integration_tests/hash_agg.rs index 36fda81f1e209..aecf0ad4d5251 100644 --- a/src/stream/tests/integration_tests/hash_agg.rs +++ b/src/stream/tests/integration_tests/hash_agg.rs @@ -39,7 +39,7 @@ async fn test_hash_agg_count_sum() { ]; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, PkIndices::new()); + let source = source.into_executor(schema, PkIndices::new()); let hash_agg = new_boxed_hash_agg_executor( store, source, @@ -118,7 +118,7 @@ async fn test_hash_agg_min() { ]; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![2]); + let source = source.into_executor(schema, vec![2]); let hash_agg = new_boxed_hash_agg_executor( store, source, @@ -194,7 +194,7 @@ async fn test_hash_agg_min_append_only() { ]; let (mut tx, source) = MockSource::channel(); - let source = source.to_executor(schema, vec![2]); + let source = source.into_executor(schema, vec![2]); let hash_agg = new_boxed_hash_agg_executor( store, source, @@ -270,7 +270,7 @@ async fn test_hash_agg_emit_on_window_close() { let create_executor = || async { let (tx, source) = MockSource::channel(); - let source = source.to_executor(input_schema.clone(), PkIndices::new()); + let source = source.into_executor(input_schema.clone(), PkIndices::new()); let hash_agg = new_boxed_hash_agg_executor( store.clone(), source, diff --git a/src/stream/tests/integration_tests/hop_window.rs b/src/stream/tests/integration_tests/hop_window.rs index a0502f2744c22..97e2886dbecda 100644 --- a/src/stream/tests/integration_tests/hop_window.rs +++ b/src/stream/tests/integration_tests/hop_window.rs @@ -25,7 +25,7 @@ const CHUNK_SIZE: usize = 256; fn create_executor(output_indices: Vec) -> (MessageSender, BoxedMessageStream) { let (tx, source) = MockSource::channel(); - let source = source.to_executor( + let source = source.into_executor( Schema::new(vec![ Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), diff --git a/src/stream/tests/integration_tests/over_window.rs b/src/stream/tests/integration_tests/over_window.rs index 2d7d8a8a13f8e..0be8e1848e9cd 100644 --- a/src/stream/tests/integration_tests/over_window.rs +++ b/src/stream/tests/integration_tests/over_window.rs @@ -74,7 +74,7 @@ async fn create_executor( .await; let (tx, source) = MockSource::channel(); - let source = source.to_executor(input_schema, input_pk_indices.clone()); + let source = source.into_executor(input_schema, input_pk_indices.clone()); let executor = OverWindowExecutor::new(OverWindowExecutorArgs { actor_ctx: ActorContext::for_test(123), diff --git a/src/stream/tests/integration_tests/project_set.rs b/src/stream/tests/integration_tests/project_set.rs index be8983cdfaa09..5cff03284f4b6 100644 --- a/src/stream/tests/integration_tests/project_set.rs +++ b/src/stream/tests/integration_tests/project_set.rs @@ -28,7 +28,7 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) { ], }; let (tx, source) = MockSource::channel(); - let source = source.to_executor(schema, PkIndices::new()); + let source = source.into_executor(schema, PkIndices::new()); let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)").into_inner(); let test_expr_watermark = build_from_pretty("(add:int8 $0:int8 1:int8)").into_inner(); From a39c4cc623a699aeddc82a24762b26a31c00ddeb Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Fri, 23 Feb 2024 13:35:19 +0800 Subject: [PATCH 11/13] fix todo Signed-off-by: Richard Chien --- src/compute/tests/cdc_tests.rs | 2 +- src/stream/src/executor/lookup_union.rs | 14 +++++--------- src/stream/src/executor/mview/materialize.rs | 2 -- src/stream/src/executor/receiver.rs | 6 +----- src/stream/src/executor/source/fetch_executor.rs | 2 -- .../src/executor/source/fs_source_executor.rs | 2 -- src/stream/src/executor/source/list_executor.rs | 2 -- src/stream/src/executor/source/source_executor.rs | 2 -- src/stream/src/executor/test_utils.rs | 6 +----- src/stream/src/executor/union.rs | 6 +----- src/stream/src/from_proto/mod.rs | 2 -- 11 files changed, 9 insertions(+), 37 deletions(-) diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs index 1347563922984..30849c40d4f84 100644 --- a/src/compute/tests/cdc_tests.rs +++ b/src/compute/tests/cdc_tests.rs @@ -232,7 +232,7 @@ async fn test_cdc_backfill() -> StreamResult<()> { ); // Create a `MaterializeExecutor` to write the changes to storage. - let materialize_table_id = TableId::new(5678); // TODO() + let materialize_table_id = TableId::new(5678); let mut materialize = MaterializeExecutor::for_test( cdc_backfill, memory_state_store.clone(), diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index bd6d48d957c67..2ffddbda45b46 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -31,15 +31,11 @@ pub struct LookupUnionExecutor { order: Vec, } -// TODO() -// impl std::fmt::Debug for LookupUnionExecutor { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// f.debug_struct("LookupUnionExecutor") -// .field("schema", &self.info.schema) -// .field("pk_indices", &self.info.pk_indices) -// .finish() -// } -// } +impl std::fmt::Debug for LookupUnionExecutor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LookupUnionExecutor").finish() + } +} impl LookupUnionExecutor { pub fn new(inputs: Vec, order: Vec) -> Self { diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index ebf68125f8c6e..c96f875765e8e 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -430,11 +430,9 @@ impl Execute for MaterializeExecutor { } } -// TODO() impl std::fmt::Debug for MaterializeExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("MaterializeExecutor") - // .field("info", &self.info) .field("arrange_key_indices", &self.arrange_key_indices) .finish() } diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index a642a581ec653..62f2dd694f910 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -49,13 +49,9 @@ pub struct ReceiverExecutor { metrics: Arc, } -// TODO() impl std::fmt::Debug for ReceiverExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ReceiverExecutor") - // .field("schema", &self.info.schema) - // .field("pk_indices", &self.info.pk_indices) - .finish() + f.debug_struct("ReceiverExecutor").finish() } } diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs index da8ad80723576..73d1b5b42f1c6 100644 --- a/src/stream/src/executor/source/fetch_executor.rs +++ b/src/stream/src/executor/source/fetch_executor.rs @@ -362,14 +362,12 @@ impl Execute for FsFetchExecutor { } } -// TODO() impl Debug for FsFetchExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if let Some(core) = &self.stream_source_core { f.debug_struct("FsFetchExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - // .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("FsFetchExecutor").finish() diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index 8eab4aaf83670..788096f6a88fc 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -482,13 +482,11 @@ impl Execute for FsSourceExecutor { } } -// TODO() impl Debug for FsSourceExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("FsSourceExecutor") .field("source_id", &self.stream_source_core.source_id) .field("column_ids", &self.stream_source_core.column_ids) - // .field("pk_indices", &self.info.pk_indices) .finish() } } diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs index f13dfdd949f0b..7996848a749e3 100644 --- a/src/stream/src/executor/source/list_executor.rs +++ b/src/stream/src/executor/source/list_executor.rs @@ -195,14 +195,12 @@ impl Execute for FsListExecutor { } } -// TODO() impl Debug for FsListExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if let Some(core) = &self.stream_source_core { f.debug_struct("FsListExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - // .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("FsListExecutor").finish() diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index ebddbd3515f57..a695b3c8b6b4b 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -622,14 +622,12 @@ impl Execute for SourceExecutor { } } -// TODO() impl Debug for SourceExecutor { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { if let Some(core) = &self.stream_source_core { f.debug_struct("SourceExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - // .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("SourceExecutor").finish() diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index 56ebd9faa1fe8..1d39041a6b7b7 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -103,13 +103,9 @@ impl MessageSender { } } -// TODO() impl std::fmt::Debug for MockSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MockSource") - // .field("schema", &self.info.schema) - // .field("pk_indices", &self.info.pk_indices) - .finish() + f.debug_struct("MockSource").finish() } } diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index f8d905a91d815..acd35ce080cda 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -31,13 +31,9 @@ pub struct UnionExecutor { inputs: Vec, } -// TODO() impl std::fmt::Debug for UnionExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("UnionExecutor") - // .field("schema", &self.info.schema) - // .field("pk_indices", &self.info.pk_indices) - .finish() + f.debug_struct("UnionExecutor").finish() } } diff --git a/src/stream/src/from_proto/mod.rs b/src/stream/src/from_proto/mod.rs index d82d27299a32a..11792feb51b4f 100644 --- a/src/stream/src/from_proto/mod.rs +++ b/src/stream/src/from_proto/mod.rs @@ -101,8 +101,6 @@ use crate::task::ExecutorParams; trait ExecutorBuilder { type Node; - // TODO(): rename - // TODO(): async trait /// Create a boxed [`Execute`] from [`StreamNode`]. fn new_boxed_executor( params: ExecutorParams, From 8840bfb13fc584a518d07bd6608f67191d3d9a39 Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Sun, 25 Feb 2024 15:18:58 +0800 Subject: [PATCH 12/13] some minor fixup Signed-off-by: Richard Chien --- src/stream/src/executor/now.rs | 4 ++-- src/stream/src/executor/test_utils.rs | 9 ++++----- src/stream/src/from_proto/mod.rs | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index be429380d598d..755e48f325965 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -52,7 +52,7 @@ impl NowExecutor { } #[try_stream(ok = Message, error = StreamExecutorError)] - async fn into_stream(self) { + async fn execute_inner(self) { let Self { data_types, barrier_receiver, @@ -151,7 +151,7 @@ impl NowExecutor { impl Execute for NowExecutor { fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() + self.execute_inner().boxed() } } diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index 1d39041a6b7b7..0be0c9b5648b1 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -110,14 +110,13 @@ impl std::fmt::Debug for MockSource { } impl MockSource { - fn new(rx: mpsc::UnboundedReceiver, stop_on_finish: bool) -> Self { - Self { rx, stop_on_finish } - } - #[allow(dead_code)] pub fn channel() -> (MessageSender, Self) { let (tx, rx) = mpsc::unbounded_channel(); - let source = Self::new(rx, true); + let source = Self { + rx, + stop_on_finish: true, + }; (MessageSender(tx), source) } diff --git a/src/stream/src/from_proto/mod.rs b/src/stream/src/from_proto/mod.rs index 11792feb51b4f..c00a9fd844df7 100644 --- a/src/stream/src/from_proto/mod.rs +++ b/src/stream/src/from_proto/mod.rs @@ -101,7 +101,7 @@ use crate::task::ExecutorParams; trait ExecutorBuilder { type Node; - /// Create a boxed [`Execute`] from [`StreamNode`]. + /// Create an [`Executor`] from [`StreamNode`]. fn new_boxed_executor( params: ExecutorParams, node: &Self::Node, From 83e68234e552d536d46fa567c8958ae75175b5fa Mon Sep 17 00:00:00 2001 From: Richard Chien Date: Mon, 26 Feb 2024 14:58:18 +0800 Subject: [PATCH 13/13] add doc comment Signed-off-by: Richard Chien --- src/stream/src/executor/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index 5c79e09891629..1e82768096f36 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -176,7 +176,7 @@ pub struct ExecutorInfo { pub identity: String, } -/// `Executor` supports handling of control messages. +/// [`Execute`] describes the methods an executor should implement to handle control messages. pub trait Execute: Send + 'static { fn execute(self: Box) -> BoxedMessageStream; @@ -192,6 +192,8 @@ pub trait Execute: Send + 'static { } } +/// [`Executor`] combines the static information ([`ExecutorInfo`]) and the executable object to +/// handle messages ([`Execute`]). pub struct Executor { info: ExecutorInfo, execute: Box,