refactor: remove associate type in scheduler to simplify it #2153 (#2194

) * feature: add a simple scheduler using flume Signed-off-by: ZhuZiyi <[email protected]> * fix: only use a sender rather clone many senders Signed-off-by: ZhuZiyi <[email protected]> * fix: use select to avoid loop Signed-off-by: ZhuZiyi <[email protected]> * feat: add parameters in new function to build the flume capacity and number of receivers Signed-off-by: ZhuZiyi <[email protected]> * test: add countdownlatch test concurrency Signed-off-by: ZhuZiyi <[email protected]> * test: add barrier replacing countdownlatch to test concurrency and add wait all tasks finished in stop Signed-off-by: ZhuZiyi <[email protected]> * chore: add some document annotation Signed-off-by: ZhuZiyi <[email protected]> * chore: add license header Signed-off-by: ZhuZiyi <[email protected]> * chore: code format Signed-off-by: ZhuZiyi <[email protected]> * chore: add Cargo.lock Signed-off-by: ZhuZiyi <[email protected]> * chore: Cargo.toml format Signed-off-by: ZhuZiyi <[email protected]> * chore: delete println in test Signed-off-by: ZhuZiyi <[email protected]> * chore: code format Signed-off-by: ZhuZiyi <[email protected]> * chore: code format Signed-off-by: ZhuZiyi <[email protected]> * feat: add error handle Signed-off-by: ZhuZiyi <[email protected]> * fix: fix error handle and add test scheduler stop Signed-off-by: ZhuZiyi <[email protected]> * chore: spelling mistake Signed-off-by: ZhuZiyi <[email protected]> * fix: wait all tasks finished Signed-off-by: ZhuZiyi <[email protected]> * chore: add todo which need wrap Future returned by send_async Signed-off-by: ZhuZiyi <[email protected]> * chore: code format Signed-off-by: ZhuZiyi <[email protected]> * test: remove unnessary sleep in test Signed-off-by: ZhuZiyi <[email protected]> * fix: resolve some conflicts Signed-off-by: ZhuZiyi <[email protected]> * fix: resolve conversation Signed-off-by: ZhuZiyi <[email protected]> * chore: code format Signed-off-by: ZhuZiyi <[email protected]> * chore: code format Signed-off-by: ZhuZiyi <[email protected]> * feat: modify the function of schedule to synchronize and drop sender after stopping scheduler Signed-off-by: ZhuZiyi <[email protected]> --------- Signed-off-by: ZhuZiyi <[email protected]>
GreptimeTeam · Aug 23, 2023 · 4dbc32f · 4dbc32f
1 parent af95e46
commit 4dbc32f
Show file tree

Hide file tree

Showing 6 changed files with 339 additions and 6 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml
@@ -33,6 +33,7 @@ dashmap = "5.4"
 datafusion-common.workspace = true
 datafusion.workspace = true
 datatypes = { workspace = true }
+flume = "0.10"
 futures.workspace = true
 lazy_static = "1.4"
 log-store = { workspace = true }
@@ -50,6 +51,7 @@ storage = { workspace = true }
 store-api = { workspace = true }
 strum = "0.21"
 table = { workspace = true }
+tokio-util.workspace = true
 tokio.workspace = true
 uuid.workspace = true
 

diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs
@@ -18,6 +18,7 @@ use std::sync::Arc;
 use common_datasource::compression::CompressionType;
 use common_error::ext::{BoxedError, ErrorExt};
 use common_error::status_code::StatusCode;
+use common_runtime::JoinError;
 use datatypes::arrow::error::ArrowError;
 use datatypes::prelude::ConcreteDataType;
 use prost::{DecodeError, EncodeError};
@@ -376,6 +377,18 @@ pub enum Error {
         source: datatypes::error::Error,
         location: Location,
     },
+
+    #[snafu(display("Invalid flume sender, location: {}", location,))]
+    InvalidFlumeSender { location: Location },
+
+    #[snafu(display("Invalid scheduler state location: {}", location,))]
+    InvalidSchedulerState { location: Location },
+
+    #[snafu(display("Failed to stop scheduler, source: {}", source))]
+    StopScheduler {
+        source: JoinError,
+        location: Location,
+    },
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -435,6 +448,9 @@ impl ErrorExt for Error {
             PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
             SortValues { .. } => StatusCode::Unexpected,
             CompactValues { source, .. } => source.status_code(),
+            InvalidFlumeSender { .. } => StatusCode::InvalidArguments,
+            InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
+            StopScheduler { .. } => StatusCode::Internal,
         }
     }
 

diff --git a/src/mito2/src/lib.rs b/src/mito2/src/lib.rs
@@ -37,6 +37,8 @@ pub mod request;
 #[allow(dead_code)]
 mod row_converter;
 #[allow(dead_code)]
+mod schedule;
+#[allow(dead_code)]
 pub mod sst;
 pub mod wal;
 #[allow(dead_code)]

diff --git a/src/mito2/src/schedule.rs b/src/mito2/src/schedule.rs
@@ -0,0 +1,15 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+pub mod scheduler;