From fdd4929c8f53c48607e01a615ada78301c95cd94 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Thu, 3 Aug 2023 12:38:46 +0900 Subject: [PATCH] refactor(mito): mv mito2 request (#2086) * refactor: mv request mod to crate level * refactor: mv SkippedFields --- src/mito2/src/engine.rs | 4 +- src/mito2/src/engine/tests.rs | 2 +- src/mito2/src/lib.rs | 2 + src/mito2/src/metadata.rs | 90 +++++++++++++-------------- src/mito2/src/{worker => }/request.rs | 0 src/mito2/src/test_util.rs | 2 +- src/mito2/src/worker.rs | 3 +- src/mito2/src/worker/handle_close.rs | 2 +- src/mito2/src/worker/handle_create.rs | 2 +- src/mito2/src/worker/handle_open.rs | 2 +- 10 files changed, 54 insertions(+), 55 deletions(-) rename src/mito2/src/{worker => }/request.rs (100%) diff --git a/src/mito2/src/engine.rs b/src/mito2/src/engine.rs index 312e25dc416d..c1ef1cb1aa69 100644 --- a/src/mito2/src/engine.rs +++ b/src/mito2/src/engine.rs @@ -26,8 +26,8 @@ use store_api::storage::RegionId; use crate::config::MitoConfig; use crate::error::{RecvSnafu, Result}; -pub use crate::worker::request::CreateRequest; -use crate::worker::request::{CloseRequest, OpenRequest, RegionRequest, RequestBody}; +pub use crate::request::CreateRequest; +use crate::request::{CloseRequest, OpenRequest, RegionRequest, RequestBody}; use crate::worker::WorkerGroup; /// Region engine implementation for timeseries data. diff --git a/src/mito2/src/engine/tests.rs b/src/mito2/src/engine/tests.rs index 87dd27e19056..f379362d5975 100644 --- a/src/mito2/src/engine/tests.rs +++ b/src/mito2/src/engine/tests.rs @@ -18,8 +18,8 @@ use store_api::storage::RegionId; use super::*; use crate::error::Error; +use crate::request::RegionOptions; use crate::test_util::{CreateRequestBuilder, TestEnv}; -use crate::worker::request::RegionOptions; #[tokio::test] async fn test_engine_new_stop() { diff --git a/src/mito2/src/lib.rs b/src/mito2/src/lib.rs index 05aa1b49a57b..b3953a2a1cd2 100644 --- a/src/mito2/src/lib.rs +++ b/src/mito2/src/lib.rs @@ -35,6 +35,8 @@ pub mod read; #[allow(dead_code)] mod region; #[allow(dead_code)] +pub mod request; +#[allow(dead_code)] pub mod sst; #[allow(dead_code)] mod worker; diff --git a/src/mito2/src/metadata.rs b/src/mito2/src/metadata.rs index 0b7162fee364..f201a88002be 100644 --- a/src/mito2/src/metadata.rs +++ b/src/mito2/src/metadata.rs @@ -119,53 +119,7 @@ impl RegionMetadata { pub fn to_json(&self) -> Result { serde_json::to_string(&self).context(SerdeJsonSnafu) } -} - -/// Fields skipped in serialization. -struct SkippedFields { - /// Last schema. - schema: SchemaRef, - /// Id of the time index column. - time_index: ColumnId, - /// Map column id to column's index in [column_metadatas](RegionMetadata::column_metadatas). - id_to_index: HashMap, -} - -impl SkippedFields { - /// Constructs skipped fields from `column_metadatas`. - fn new(column_metadatas: &[ColumnMetadata]) -> Result { - let column_schemas = column_metadatas - .iter() - .map(|column_metadata| column_metadata.column_schema.clone()) - .collect(); - let schema = Arc::new(Schema::try_new(column_schemas).context(InvalidSchemaSnafu)?); - let time_index = column_metadatas - .iter() - .find_map(|col| { - if col.semantic_type == SemanticType::Timestamp { - Some(col.column_id) - } else { - None - } - }) - .context(InvalidMetaSnafu { - reason: "time index not found", - })?; - let id_to_index = column_metadatas - .iter() - .enumerate() - .map(|(idx, col)| (col.column_id, idx)) - .collect(); - - Ok(SkippedFields { - schema, - time_index, - id_to_index, - }) - } -} -impl RegionMetadata { /// Find column by id. pub(crate) fn column_by_id(&self, column_id: ColumnId) -> Option<&ColumnMetadata> { self.id_to_index @@ -366,6 +320,50 @@ pub enum SemanticType { Timestamp, } +/// Fields skipped in serialization. +struct SkippedFields { + /// Last schema. + schema: SchemaRef, + /// Id of the time index column. + time_index: ColumnId, + /// Map column id to column's index in [column_metadatas](RegionMetadata::column_metadatas). + id_to_index: HashMap, +} + +impl SkippedFields { + /// Constructs skipped fields from `column_metadatas`. + fn new(column_metadatas: &[ColumnMetadata]) -> Result { + let column_schemas = column_metadatas + .iter() + .map(|column_metadata| column_metadata.column_schema.clone()) + .collect(); + let schema = Arc::new(Schema::try_new(column_schemas).context(InvalidSchemaSnafu)?); + let time_index = column_metadatas + .iter() + .find_map(|col| { + if col.semantic_type == SemanticType::Timestamp { + Some(col.column_id) + } else { + None + } + }) + .context(InvalidMetaSnafu { + reason: "time index not found", + })?; + let id_to_index = column_metadatas + .iter() + .enumerate() + .map(|(idx, col)| (col.column_id, idx)) + .collect(); + + Ok(SkippedFields { + schema, + time_index, + id_to_index, + }) + } +} + #[cfg(test)] mod test { use datatypes::prelude::ConcreteDataType; diff --git a/src/mito2/src/worker/request.rs b/src/mito2/src/request.rs similarity index 100% rename from src/mito2/src/worker/request.rs rename to src/mito2/src/request.rs diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs index 4f72059cc60d..837041edc7ba 100644 --- a/src/mito2/src/test_util.rs +++ b/src/mito2/src/test_util.rs @@ -33,7 +33,7 @@ use crate::error::Result; use crate::manifest::manager::RegionManifestManager; use crate::manifest::options::RegionManifestOptions; use crate::metadata::{ColumnMetadata, RegionMetadataRef, SemanticType}; -use crate::worker::request::{CreateRequest, RegionOptions}; +use crate::request::{CreateRequest, RegionOptions}; use crate::worker::WorkerGroup; /// Env to test mito engine. diff --git a/src/mito2/src/worker.rs b/src/mito2/src/worker.rs index c6ecfb12435b..874a76d52d2d 100644 --- a/src/mito2/src/worker.rs +++ b/src/mito2/src/worker.rs @@ -17,7 +17,6 @@ mod handle_close; mod handle_create; mod handle_open; -pub(crate) mod request; use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; @@ -38,7 +37,7 @@ use crate::config::MitoConfig; use crate::error::{JoinSnafu, Result, WorkerStoppedSnafu}; use crate::memtable::{DefaultMemtableBuilder, MemtableBuilderRef}; use crate::region::{RegionMap, RegionMapRef}; -use crate::worker::request::{RegionRequest, RequestBody, WorkerRequest}; +use crate::request::{RegionRequest, RequestBody, WorkerRequest}; /// Identifier for a worker. pub(crate) type WorkerId = u32; diff --git a/src/mito2/src/worker/handle_close.rs b/src/mito2/src/worker/handle_close.rs index d3f6c4585116..f08aa4ad59f8 100644 --- a/src/mito2/src/worker/handle_close.rs +++ b/src/mito2/src/worker/handle_close.rs @@ -17,7 +17,7 @@ use common_telemetry::info; use crate::error::Result; -use crate::worker::request::CloseRequest; +use crate::request::CloseRequest; use crate::worker::RegionWorkerLoop; impl RegionWorkerLoop { diff --git a/src/mito2/src/worker/handle_create.rs b/src/mito2/src/worker/handle_create.rs index 62f0dcb1e547..75a2cfd3bf61 100644 --- a/src/mito2/src/worker/handle_create.rs +++ b/src/mito2/src/worker/handle_create.rs @@ -22,7 +22,7 @@ use snafu::ensure; use crate::error::{RegionExistsSnafu, Result}; use crate::metadata::{RegionMetadataBuilder, INIT_REGION_VERSION}; use crate::region::opener::RegionOpener; -use crate::worker::request::CreateRequest; +use crate::request::CreateRequest; use crate::worker::RegionWorkerLoop; impl RegionWorkerLoop { diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs index 3128f5fa968c..9cca66bc7bce 100644 --- a/src/mito2/src/worker/handle_open.rs +++ b/src/mito2/src/worker/handle_open.rs @@ -20,7 +20,7 @@ use common_telemetry::info; use crate::error::Result; use crate::region::opener::RegionOpener; -use crate::worker::request::OpenRequest; +use crate::request::OpenRequest; use crate::worker::RegionWorkerLoop; impl RegionWorkerLoop {