Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into rfc-partition-rule
Browse files Browse the repository at this point in the history
waynexia committed Feb 22, 2024
2 parents 726c93d + 8289b0d commit 7d86fc8
Showing 130 changed files with 2,558 additions and 287 deletions.
10 changes: 0 additions & 10 deletions .cargo/config.toml
Original file line number Diff line number Diff line change
@@ -3,13 +3,3 @@ linker = "aarch64-linux-gnu-gcc"

[alias]
sqlness = "run --bin sqlness-runner --"


[build]
rustflags = [
# lints
# TODO: use lint configuration in cargo https://github.com/rust-lang/cargo/issues/5034
"-Wclippy::print_stdout",
"-Wclippy::print_stderr",
"-Wclippy::implicit_clone",
]
14 changes: 13 additions & 1 deletion .github/workflows/docs.yml
Original file line number Diff line number Diff line change
@@ -61,6 +61,18 @@ jobs:

sqlness:
name: Sqlness Test
runs-on: ubuntu-20.04
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-20.04 ]
steps:
- run: 'echo "No action required"'

sqlness-kafka-wal:
name: Sqlness Test with Kafka Wal
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ ubuntu-20.04 ]
steps:
- run: 'echo "No action required"'
11 changes: 11 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 5 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -66,6 +66,11 @@ version = "0.6.0"
edition = "2021"
license = "Apache-2.0"

[workspace.lints]
clippy.print_stdout = "warn"
clippy.print_sterr = "warn"
clippy.implicit_clone = "warn"

[workspace.dependencies]
ahash = { version = "0.8", features = ["compile-time-rng"] }
aquamarine = "0.3"
3 changes: 3 additions & 0 deletions benchmarks/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
arrow.workspace = true
chrono.workspace = true
3 changes: 3 additions & 0 deletions src/api/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
common-base.workspace = true
common-decimal.workspace = true
3 changes: 3 additions & 0 deletions src/auth/Cargo.toml
Original file line number Diff line number Diff line change
@@ -8,6 +8,9 @@ license.workspace = true
default = []
testing = []

[lints]
workspace = true

[dependencies]
api.workspace = true
async-trait.workspace = true
3 changes: 3 additions & 0 deletions src/catalog/Cargo.toml
Original file line number Diff line number Diff line change
@@ -7,6 +7,9 @@ license.workspace = true
[features]
testing = []

[lints]
workspace = true

[dependencies]
api.workspace = true
arc-swap = "1.0"
7 changes: 2 additions & 5 deletions src/catalog/src/error.rs
Original file line number Diff line number Diff line change
@@ -164,11 +164,8 @@ pub enum Error {
location: Location,
},

#[snafu(display("Failed to find table partitions: #{table}"))]
FindPartitions {
source: partition::error::Error,
table: String,
},
#[snafu(display("Failed to find table partitions"))]
FindPartitions { source: partition::error::Error },

#[snafu(display("Failed to find region routes"))]
FindRegionRoutes { source: partition::error::Error },
81 changes: 53 additions & 28 deletions src/catalog/src/information_schema/partitions.rs
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

use core::pin::pin;
use std::sync::{Arc, Weak};

use arrow_schema::SchemaRef as ArrowSchemaRef;
@@ -31,7 +32,7 @@ use datatypes::vectors::{
ConstantVector, DateTimeVector, DateTimeVectorBuilder, Int64Vector, Int64VectorBuilder,
MutableVector, StringVector, StringVectorBuilder, UInt64VectorBuilder,
};
use futures::TryStreamExt;
use futures::{StreamExt, TryStreamExt};
use partition::manager::PartitionInfo;
use partition::partition::PartitionDef;
use snafu::{OptionExt, ResultExt};
@@ -240,40 +241,64 @@ impl InformationSchemaPartitionsBuilder {
let predicates = Predicates::from_scan_request(&request);

for schema_name in catalog_manager.schema_names(&catalog_name).await? {
let mut stream = catalog_manager.tables(&catalog_name, &schema_name).await;

while let Some(table) = stream.try_next().await? {
let table_info = table.table_info();

if table_info.table_type == TableType::Temporary {
continue;
}

let table_id = table_info.ident.table_id;
let partitions = if let Some(partition_manager) = &partition_manager {
let table_info_stream = catalog_manager
.tables(&catalog_name, &schema_name)
.await
.try_filter_map(|t| async move {
let table_info = t.table_info();
if table_info.table_type == TableType::Temporary {
Ok(None)
} else {
Ok(Some(table_info))
}
});

const BATCH_SIZE: usize = 128;

// Split table infos into chunks
let mut table_info_chunks = pin!(table_info_stream.ready_chunks(BATCH_SIZE));

while let Some(table_infos) = table_info_chunks.next().await {
let table_infos = table_infos.into_iter().collect::<Result<Vec<_>>>()?;
let table_ids: Vec<TableId> =
table_infos.iter().map(|info| info.ident.table_id).collect();

let mut table_partitions = if let Some(partition_manager) = &partition_manager {
partition_manager
.find_table_partitions(table_id)
.batch_find_table_partitions(&table_ids)
.await
.context(FindPartitionsSnafu {
table: &table_info.name,
})?
.context(FindPartitionsSnafu)?
} else {
// Current node must be a standalone instance, contains only one partition by default.
// TODO(dennis): change it when we support multi-regions for standalone.
vec![PartitionInfo {
id: RegionId::new(table_id, 0),
partition: PartitionDef::new(vec![], vec![]),
}]
table_ids
.into_iter()
.map(|table_id| {
(
table_id,
vec![PartitionInfo {
id: RegionId::new(table_id, 0),
partition: PartitionDef::new(vec![], vec![]),
}],
)
})
.collect()
};

self.add_partitions(
&predicates,
&table_info,
&catalog_name,
&schema_name,
&table_info.name,
&partitions,
);
for table_info in table_infos {
let partitions = table_partitions
.remove(&table_info.ident.table_id)
.unwrap_or(vec![]);

self.add_partitions(
&predicates,
&table_info,
&catalog_name,
&schema_name,
&table_info.name,
&partitions,
);
}
}
}

2 changes: 1 addition & 1 deletion src/catalog/src/information_schema/region_peers.rs
Original file line number Diff line number Diff line change
@@ -199,7 +199,7 @@ impl InformationSchemaRegionPeersBuilder {

let table_routes = if let Some(partition_manager) = &partition_manager {
partition_manager
.find_region_routes_batch(&table_ids)
.batch_find_region_routes(&table_ids)
.await
.context(FindRegionRoutesSnafu)?
} else {
3 changes: 3 additions & 0 deletions src/client/Cargo.toml
Original file line number Diff line number Diff line change
@@ -7,6 +7,9 @@ license.workspace = true
[features]
testing = []

[lints]
workspace = true

[dependencies]
api.workspace = true
arc-swap = "1.6"
3 changes: 3 additions & 0 deletions src/cmd/Cargo.toml
Original file line number Diff line number Diff line change
@@ -12,6 +12,9 @@ path = "src/bin/greptime.rs"
[features]
tokio-console = ["common-telemetry/tokio-console"]

[lints]
workspace = true

[dependencies]
anymap = "1.0.0-beta.2"
async-trait.workspace = true
1 change: 1 addition & 0 deletions src/cmd/src/cli/repl.rs
Original file line number Diff line number Diff line change
@@ -260,6 +260,7 @@ async fn create_query_engine(meta_addr: &str) -> Result<DatafusionQueryEngine> {
catalog_list,
None,
None,
None,
false,
plugins.clone(),
));
8 changes: 4 additions & 4 deletions src/cmd/src/standalone.rs
Original file line number Diff line number Diff line change
@@ -22,7 +22,7 @@ use common_config::{metadata_store_dir, KvBackendConfig};
use common_meta::cache_invalidator::DummyCacheInvalidator;
use common_meta::datanode_manager::DatanodeManagerRef;
use common_meta::ddl::table_meta::{TableMetadataAllocator, TableMetadataAllocatorRef};
use common_meta::ddl::DdlTaskExecutorRef;
use common_meta::ddl::ProcedureExecutorRef;
use common_meta::ddl_manager::DdlManager;
use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
use common_meta::kv_backend::KvBackendRef;
@@ -459,8 +459,8 @@ impl StartCommand {
procedure_manager: ProcedureManagerRef,
datanode_manager: DatanodeManagerRef,
table_meta_allocator: TableMetadataAllocatorRef,
) -> Result<DdlTaskExecutorRef> {
let ddl_task_executor: DdlTaskExecutorRef = Arc::new(
) -> Result<ProcedureExecutorRef> {
let procedure_executor: ProcedureExecutorRef = Arc::new(
DdlManager::try_new(
procedure_manager,
datanode_manager,
@@ -472,7 +472,7 @@ impl StartCommand {
.context(InitDdlManagerSnafu)?,
);

Ok(ddl_task_executor)
Ok(procedure_executor)
}

pub async fn create_table_metadata_manager(
3 changes: 3 additions & 0 deletions src/common/base/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
anymap = "1.0.0-beta.2"
bitvec = "1.0"
3 changes: 3 additions & 0 deletions src/common/catalog/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
common-error.workspace = true
common-macro.workspace = true
3 changes: 3 additions & 0 deletions src/common/config/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
common-base.workspace = true
humantime-serde.workspace = true
3 changes: 3 additions & 0 deletions src/common/datasource/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
arrow.workspace = true
arrow-schema.workspace = true
3 changes: 3 additions & 0 deletions src/common/decimal/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
arrow.workspace = true
bigdecimal.workspace = true
3 changes: 3 additions & 0 deletions src/common/error/Cargo.toml
Original file line number Diff line number Diff line change
@@ -4,6 +4,9 @@ version.workspace = true
edition.workspace = true
license.workspace = true

[lints]
workspace = true

[dependencies]
snafu.workspace = true
strum.workspace = true
6 changes: 4 additions & 2 deletions src/common/error/src/lib.rs
Original file line number Diff line number Diff line change
@@ -19,7 +19,9 @@ pub mod format;
pub mod mock;
pub mod status_code;

pub use snafu;

// HACK - these headers are here for shared in gRPC services. For common HTTP headers,
// please define in `src/servers/src/http/header.rs`.
pub const GREPTIME_DB_HEADER_ERROR_CODE: &str = "x-greptime-err-code";
pub const GREPTIME_DB_HEADER_ERROR_MSG: &str = "x-greptime-err-msg";

pub use snafu;
Loading

0 comments on commit 7d86fc8

Please sign in to comment.