From 026f34b41ea67f1dc13ed677671f31b3867e8d34 Mon Sep 17 00:00:00 2001 From: Kould <2435992353@qq.com> Date: Wed, 27 Sep 2023 20:54:28 +0800 Subject: [PATCH] refactor: reconstruct the Key encoding of each structure of TableCodec https://github.com/KipData/KipSQL/issues/68 --- src/catalog/root.rs | 3 +- src/catalog/table.rs | 32 +- src/execution/executor/show/show_table.rs | 30 +- src/storage/kip.rs | 102 ++--- src/storage/memory.rs | 2 +- src/storage/mod.rs | 2 +- src/storage/table_codec.rs | 517 +++++++++++----------- src/types/errors.rs | 7 + 8 files changed, 368 insertions(+), 327 deletions(-) diff --git a/src/catalog/root.rs b/src/catalog/root.rs index d7e8981c..b047b0dd 100644 --- a/src/catalog/root.rs +++ b/src/catalog/root.rs @@ -35,8 +35,7 @@ impl RootCatalog { } let table = TableCatalog::new( table_name.clone(), - columns, - vec![] + columns )?; self.table_idxs.insert(table_name.clone(), table); diff --git a/src/catalog/table.rs b/src/catalog/table.rs index 959e8025..76a19a24 100644 --- a/src/catalog/table.rs +++ b/src/catalog/table.rs @@ -1,6 +1,5 @@ use std::collections::BTreeMap; use std::sync::Arc; -use itertools::Itertools; use crate::catalog::{CatalogError, ColumnCatalog, ColumnRef}; use crate::types::ColumnId; @@ -73,20 +72,24 @@ impl TableCatalog { Ok(col_id) } + pub(crate) fn add_index_meta(&mut self, mut index: IndexMeta) -> &IndexMeta { + let index_id = self.indexes.len(); + + index.id = index_id as u32; + self.indexes.push(Arc::new(index)); + + &self.indexes[index_id] + } + pub(crate) fn new( name: TableName, - columns: Vec, - indexes: Vec + columns: Vec ) -> Result { - let indexes = indexes.into_iter() - .map(Arc::new) - .collect_vec(); - let mut table_catalog = TableCatalog { name, column_idxs: BTreeMap::new(), columns: BTreeMap::new(), - indexes, + indexes: vec![], }; for col_catalog in columns.into_iter() { @@ -95,6 +98,17 @@ impl TableCatalog { Ok(table_catalog) } + + pub(crate) fn new_with_indexes( + name: TableName, + columns: Vec, + indexes: Vec + ) -> Result { + let mut catalog = TableCatalog::new(name, columns)?; + catalog.indexes = indexes; + + Ok(catalog) + } } #[cfg(test)] @@ -112,7 +126,7 @@ mod tests { let col0 = ColumnCatalog::new("a".into(), false, ColumnDesc::new(LogicalType::Integer, false, false)); let col1 = ColumnCatalog::new("b".into(), false, ColumnDesc::new(LogicalType::Boolean, false, false)); let col_catalogs = vec![col0, col1]; - let table_catalog = TableCatalog::new(Arc::new("test".to_string()), col_catalogs, vec![]).unwrap(); + let table_catalog = TableCatalog::new(Arc::new("test".to_string()), col_catalogs).unwrap(); assert_eq!(table_catalog.contains_column(&"a".to_string()), true); assert_eq!(table_catalog.contains_column(&"b".to_string()), true); diff --git a/src/execution/executor/show/show_table.rs b/src/execution/executor/show/show_table.rs index 4ca19c01..8641cc0f 100644 --- a/src/execution/executor/show/show_table.rs +++ b/src/execution/executor/show/show_table.rs @@ -30,23 +30,21 @@ impl Executor for ShowTables { impl ShowTables { #[try_stream(boxed, ok = Tuple, error = ExecutorError)] pub async fn _execute(self, storage: S) { - if let Some(tables) = storage.show_tables().await { - for (table,column_count) in tables { - let columns: Vec = vec![ - Arc::new(ColumnCatalog::new_dummy("TABLES".to_string())), - Arc::new(ColumnCatalog::new_dummy("COLUMN_COUNT".to_string())), - ]; - let values: Vec = vec![ - Arc::new(DataValue::Utf8(Some(table))), - Arc::new(DataValue::UInt32(Some(column_count as u32))), - ]; + let tables = storage.show_tables().await?; - yield Tuple { - id: None, - columns, - values, - }; - } + for table in tables { + let columns: Vec = vec![ + Arc::new(ColumnCatalog::new_dummy("TABLES".to_string())), + ]; + let values: Vec = vec![ + Arc::new(DataValue::Utf8(Some(table))), + ]; + + yield Tuple { + id: None, + columns, + values, + }; } } } \ No newline at end of file diff --git a/src/storage/kip.rs b/src/storage/kip.rs index 0facf1f6..bb08d65c 100644 --- a/src/storage/kip.rs +++ b/src/storage/kip.rs @@ -41,28 +41,30 @@ impl KipStorage { }) } - fn column_collect(name: &String, tx: &mvcc::Transaction) -> Option<(Vec, Option)> { + fn column_collect(name: &String, tx: &mvcc::Transaction) -> Result<(Vec, Option), StorageError> { let (column_min, column_max) = TableCodec::columns_bound(name); - let mut column_iter = tx.iter(Bound::Included(&column_min), Bound::Included(&column_max)).ok()?; + let mut column_iter = tx.iter(Bound::Included(&column_min), Bound::Included(&column_max))?; let mut columns = vec![]; let mut name_option = None; - while let Some((key, value_option)) = column_iter.try_next().ok().flatten() { + while let Some((_, value_option)) = column_iter.try_next().ok().flatten() { if let Some(value) = value_option { - if let Some((table_name, column)) = TableCodec::decode_column(&key, &value) { - if name != table_name.as_str() { return None; } - let _ = name_option.insert(table_name); + let (table_name, column) = TableCodec::decode_column(&value)?; - columns.push(column); + if name != table_name.as_str() { + return Ok((vec![], None)); } + let _ = name_option.insert(table_name); + + columns.push(column); } } - Some((columns, name_option)) + Ok((columns, name_option)) } - fn index_meta_collect(name: &String, tx: &mvcc::Transaction) -> Option> { + fn index_meta_collect(name: &String, tx: &mvcc::Transaction) -> Option> { let (index_min, index_max) = TableCodec::index_meta_bound(name); let mut index_metas = vec![]; let mut index_iter = tx.iter(Bound::Included(&index_min), Bound::Included(&index_max)).ok()?; @@ -70,7 +72,7 @@ impl KipStorage { while let Some((_, value_option)) = index_iter.try_next().ok().flatten() { if let Some(value) = value_option { if let Some(index_meta) = TableCodec::decode_index_meta(&value).ok() { - index_metas.push(index_meta); + index_metas.push(Arc::new(index_meta)); } } } @@ -95,54 +97,54 @@ impl KipStorage { Ok(()) } -} - -#[async_trait] -impl Storage for KipStorage { - type TransactionType = KipTransaction; - - async fn create_table(&self, table_name: TableName, mut columns: Vec) -> Result { - let mut tx = self.inner.new_transaction().await; - for (i, col) in columns.iter_mut().enumerate() { - col.id = Some(i as u32); - } - - for (key, value) in columns - .iter() - .filter_map(|col| TableCodec::encode_column(&table_name, &col)) - { - tx.set(key, value); - } - let mut indexes = Vec::new(); + fn create_index_meta_for_table( + tx: &mut mvcc::Transaction, + table: &mut TableCatalog + ) -> Result<(), StorageError> { + let table_name = table.name.clone(); - for col in columns - .iter() + for col in table.all_columns() + .into_iter() .filter(|col| col.desc.is_unique) { if let Some(col_id) = col.id { let meta = IndexMeta { - id: indexes.len() as u32, + id: 0, column_ids: vec![col_id], name: format!("uk_{}", col.name), is_unique: true, }; - let (key, value) = TableCodec::encode_index_meta(&table_name, &meta)?; + let meta_ref = table.add_index_meta(meta); + let (key, value) = TableCodec::encode_index_meta(&table_name, meta_ref)?; - indexes.push(meta); tx.set(key, value); } } + Ok(()) + } +} + +#[async_trait] +impl Storage for KipStorage { + type TransactionType = KipTransaction; + + async fn create_table(&self, table_name: TableName, columns: Vec) -> Result { + let mut tx = self.inner.new_transaction().await; + let mut table_catalog = TableCatalog::new(table_name.clone(), columns)?; + + Self::create_index_meta_for_table(&mut tx, &mut table_catalog)?; - let (k, v)= TableCodec::encode_root_table(table_name.as_str(), columns.len())?; + for (_, column) in &table_catalog.columns { + let (key, value) = TableCodec::encode_column(column)?; + tx.set(key, value); + } + + let (k, v)= TableCodec::encode_root_table(&table_name)?; self.inner.set(k, v).await?; tx.commit().await?; - - self.cache.put( - table_name.to_string(), - TableCatalog::new(table_name.clone(), columns, indexes)? - ); + self.cache.put(table_name.to_string(), table_catalog); Ok(table_name) } @@ -165,7 +167,7 @@ impl Storage for KipStorage { for col_key in col_keys { tx.remove(&col_key)? } - tx.remove(&TableCodec::encode_root_table_key(name.as_str()))?; + tx.remove(&TableCodec::encode_root_table_key(name))?; tx.commit().await?; let _ = self.cache.remove(name); @@ -203,11 +205,11 @@ impl Storage for KipStorage { if option.is_none() { let tx = self.inner.new_transaction().await; // TODO: unify the data into a `Meta` prefix and use one iteration to collect all data - let (columns, name_option) = Self::column_collect(name, &tx)?; + let (columns, name_option) = Self::column_collect(name, &tx).ok()?; let indexes = Self::index_meta_collect(name, &tx)?; if let Some(catalog) = name_option - .and_then(|table_name| TableCatalog::new(table_name, columns, indexes).ok()) + .and_then(|table_name| TableCatalog::new_with_indexes(table_name, columns, indexes).ok()) { option = self.cache.get_or_insert(name.to_string(), |_| Ok(catalog)).ok(); } @@ -216,22 +218,22 @@ impl Storage for KipStorage { option } - async fn show_tables(&self) -> Option> { + async fn show_tables(&self) -> Result, StorageError> { let mut tables = vec![]; let (min, max) = TableCodec::root_table_bound(); let tx = self.inner.new_transaction().await; - let mut iter = tx.iter(Bound::Included(&min), Bound::Included(&max)).ok()?; + let mut iter = tx.iter(Bound::Included(&min), Bound::Included(&max))?; - while let Some((key, value_option)) = iter.try_next().ok().flatten() { + while let Some((_, value_option)) = iter.try_next().ok().flatten() { if let Some(value) = value_option { - if let Some((table_name, column_count)) = TableCodec::decode_root_table(&key, &value) { - tables.push((table_name,column_count)); - } + let table_name = TableCodec::decode_root_table(&value)?; + + tables.push(table_name); } } - Some(tables) + Ok(tables) } } diff --git a/src/storage/memory.rs b/src/storage/memory.rs index 8597aeff..aaf241a8 100644 --- a/src/storage/memory.rs +++ b/src/storage/memory.rs @@ -117,7 +117,7 @@ impl Storage for MemStorage { } } - async fn show_tables(&self) -> Option> { + async fn show_tables(&self) -> Result, StorageError> { todo!() } } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 20b05daf..553e9662 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -32,7 +32,7 @@ pub trait Storage: Sync + Send + Clone + 'static { async fn transaction(&self, name: &String) -> Option; async fn table(&self, name: &String) -> Option<&TableCatalog>; - async fn show_tables(&self) -> Option>; + async fn show_tables(&self) -> Result, StorageError>; } /// Optional bounds of the reader, of the form (offset, limit). diff --git a/src/storage/table_codec.rs b/src/storage/table_codec.rs index 49edc50d..e8afa7fa 100644 --- a/src/storage/table_codec.rs +++ b/src/storage/table_codec.rs @@ -1,5 +1,5 @@ -use std::sync::Arc; use bytes::Bytes; +use lazy_static::lazy_static; use crate::catalog::{ColumnCatalog, TableCatalog, TableName}; use crate::types::errors::TypeError; use crate::types::index::{Index, IndexId, IndexMeta}; @@ -7,74 +7,126 @@ use crate::types::tuple::{Tuple, TupleId}; const BOUND_MIN_TAG: u8 = 0; const BOUND_MAX_TAG: u8 = 1; - -const COLUMNS_ID_LEN: usize = 10; +lazy_static! { + static ref ROOT_BYTES: Vec = { + b"Root".to_vec() + }; +} #[derive(Clone)] pub struct TableCodec { pub table: TableCatalog } +#[derive(Copy, Clone)] +enum CodecType { + Column, + IndexMeta, + Index, + Tuple, + Root, +} + impl TableCodec { + /// TableName + Type + /// + /// Tips: Root full key = key_prefix + fn key_prefix(ty: CodecType, table_name: &String) -> Vec { + let mut table_bytes = table_name + .clone() + .into_bytes(); + + match ty { + CodecType::Column => { + table_bytes.push(b'0'); + } + CodecType::IndexMeta => { + table_bytes.push(b'1'); + } + CodecType::Index => { + table_bytes.push(b'2'); + } + CodecType::Tuple => { + table_bytes.push(b'3'); + } + CodecType::Root => { + let mut bytes = ROOT_BYTES.clone(); + bytes.push(BOUND_MIN_TAG); + bytes.append(&mut table_bytes); + + table_bytes = bytes + } + } + + table_bytes + } + pub fn tuple_bound(&self) -> (Vec, Vec) { let op = |bound_id| { - format!( - "{}_Tuple_{}", - self.table.name, - bound_id - ) + let mut key_prefix = Self::key_prefix(CodecType::Tuple, &self.table.name); + + key_prefix.push(bound_id); + key_prefix }; - (op(BOUND_MIN_TAG).into_bytes(), op(BOUND_MAX_TAG).into_bytes()) + (op(BOUND_MIN_TAG), op(BOUND_MAX_TAG)) } pub fn index_meta_bound(name: &String) -> (Vec, Vec) { let op = |bound_id| { - format!( - "{}_IndexMeta_{}", - name, - bound_id - ) + let mut key_prefix = Self::key_prefix(CodecType::IndexMeta, name); + + key_prefix.push(bound_id); + key_prefix }; - (op(BOUND_MIN_TAG).into_bytes(), op(BOUND_MAX_TAG).into_bytes()) + (op(BOUND_MIN_TAG), op(BOUND_MAX_TAG)) } pub fn index_bound(&self, index_id: &IndexId) -> (Vec, Vec) { let op = |bound_id| { - format!( - "{}_Index_0_{}_{}", - self.table.name, - index_id, - bound_id - ) + let mut key_prefix = Self::key_prefix(CodecType::Index, &self.table.name); + + key_prefix.push(BOUND_MIN_TAG); + key_prefix.append(&mut index_id.to_be_bytes().to_vec()); + key_prefix.push(bound_id); + key_prefix }; - (op(BOUND_MIN_TAG).into_bytes(), op(BOUND_MAX_TAG).into_bytes()) + (op(BOUND_MIN_TAG), op(BOUND_MAX_TAG)) } pub fn all_index_bound(&self) -> (Vec, Vec) { let op = |bound_id| { - format!( - "{}_Index_{}", - self.table.name, - bound_id - ) + let mut key_prefix = Self::key_prefix(CodecType::Index, &self.table.name); + + key_prefix.push(bound_id); + key_prefix + }; + + (op(BOUND_MIN_TAG), op(BOUND_MAX_TAG)) + } + + pub fn root_table_bound() -> (Vec, Vec) { + let op = |bound_id| { + let mut key_prefix = ROOT_BYTES.clone(); + + key_prefix.push(bound_id); + key_prefix }; - (op(BOUND_MIN_TAG).into_bytes(), op(BOUND_MAX_TAG).into_bytes()) + (op(BOUND_MIN_TAG), op(BOUND_MAX_TAG)) } pub fn columns_bound(name: &String) -> (Vec, Vec) { let op = |bound_id| { - format!( - "{}_Catalog_{}", - name, - bound_id - ) + let mut key_prefix = Self::key_prefix(CodecType::Column, &name); + + key_prefix.push(bound_id); + key_prefix }; - (op(BOUND_MIN_TAG).into_bytes(), op(BOUND_MAX_TAG).into_bytes()) + (op(BOUND_MIN_TAG), op(BOUND_MAX_TAG)) } /// Key: TableName_Tuple_0_RowID(Sorted) @@ -90,14 +142,12 @@ impl TableCodec { } pub fn encode_tuple_key(&self, tuple_id: &TupleId) -> Result, TypeError> { - let mut string_key = format!( - "{}_Tuple_0_", - self.table.name, - ).into_bytes(); + let mut key_prefix = Self::key_prefix(CodecType::Tuple, &self.table.name); + key_prefix.push(BOUND_MIN_TAG); - tuple_id.to_primary_key(&mut string_key)?; + tuple_id.to_primary_key(&mut key_prefix)?; - Ok(string_key) + Ok(key_prefix) } pub fn decode_tuple(&self, bytes: &[u8]) -> Tuple { @@ -107,13 +157,11 @@ impl TableCodec { /// Key: TableName_IndexMeta_0_IndexID /// Value: IndexMeta pub fn encode_index_meta(name: &String, index_meta: &IndexMeta) -> Result<(Bytes, Bytes), TypeError> { - let key = format!( - "{}_IndexMeta_0_{}", - name, - index_meta.id - ); + let mut key_prefix = Self::key_prefix(CodecType::IndexMeta, &name); + key_prefix.push(BOUND_MIN_TAG); + key_prefix.append(&mut index_meta.id.to_be_bytes().to_vec()); - Ok((Bytes::from(key), Bytes::from(bincode::serialize(&index_meta)?))) + Ok((Bytes::from(key_prefix), Bytes::from(bincode::serialize(&index_meta)?))) } pub fn decode_index_meta(bytes: &[u8]) -> Result { @@ -137,17 +185,16 @@ impl TableCodec { } pub fn encode_index_key(&self, index: &Index) -> Result, TypeError> { - let mut string_key = format!( - "{}_Index_0_{}_0", - self.table.name, - index.id - ).into_bytes(); + let mut key_prefix = Self::key_prefix(CodecType::Index, &self.table.name); + key_prefix.push(BOUND_MIN_TAG); + key_prefix.append(&mut index.id.to_be_bytes().to_vec()); + key_prefix.push(BOUND_MIN_TAG); for col_v in &index.column_values { - col_v.to_index_key(&mut string_key)?; + col_v.to_index_key(&mut key_prefix)?; } - Ok(string_key) + Ok(key_prefix) } pub fn decode_index(bytes: &[u8]) -> Result, TypeError> { @@ -158,74 +205,37 @@ impl TableCodec { /// Value: ColumnCatalog /// /// Tips: the `0` for bound range - pub fn encode_column(table_name: &String, col: &ColumnCatalog) -> Option<(Bytes, Bytes)> { - bincode::serialize(col).ok() - .map(|bytes| { - let key = format!( - "{}_Catalog_{}_{}_{:0width$}", - table_name, - BOUND_MIN_TAG, - col.name, - col.id.unwrap(), - width = COLUMNS_ID_LEN - ); - - (Bytes::from(key.into_bytes()), Bytes::from(bytes)) - }) - } - - pub fn decode_column(key: &[u8], bytes: &[u8]) -> Option<(TableName, ColumnCatalog)> { - String::from_utf8(key.to_owned()).ok()? - .split("_") - .nth(0) - .and_then(|table_name| { - bincode::deserialize::(bytes).ok() - .and_then(|col| { - Some((Arc::new(table_name.to_string()), col)) - }) - }) + pub fn encode_column(col: &ColumnCatalog) -> Result<(Bytes, Bytes), TypeError> { + let bytes = bincode::serialize(col)?; + let mut key_prefix = Self::key_prefix(CodecType::Column, col.table_name.as_ref().unwrap()); + + key_prefix.push(BOUND_MIN_TAG); + key_prefix.append(&mut col.id.unwrap().to_be_bytes().to_vec()); + + Ok((Bytes::from(key_prefix), Bytes::from(bytes))) + } + + pub fn decode_column(bytes: &[u8]) -> Result<(TableName, ColumnCatalog), TypeError> { + let column = bincode::deserialize::(bytes)?; + + Ok((column.table_name.clone().unwrap(), column)) } /// Key: RootCatalog_0_TableName - /// Value: ColumnCount - pub fn encode_root_table(table_name: &str,column_count:usize) -> Result<(Bytes, Bytes), TypeError> { + /// Value: TableName + pub fn encode_root_table(table_name: &String) -> Result<(Bytes, Bytes), TypeError> { let key = Self::encode_root_table_key(table_name); - let bytes = bincode::serialize(&column_count)?; - Ok((Bytes::from(key), Bytes::from(bytes))) + Ok((Bytes::from(key), Bytes::from(table_name.clone().into_bytes()))) } - pub fn encode_root_table_key(table_name: &str) -> Vec { - format!( - "RootCatalog_{}_{}", - BOUND_MIN_TAG, - table_name, - ).into_bytes() + pub fn encode_root_table_key(table_name: &String) -> Vec { + Self::key_prefix(CodecType::Root, &table_name) } - // TODO: value is reserved for saving meta-information - pub fn decode_root_table(key: &[u8], bytes: &[u8]) -> Option<(String,usize)> { - String::from_utf8(key.to_owned()).ok()? - .split("_") - .nth(2) - .and_then(|table_name| { - bincode::deserialize::(bytes).ok() - .and_then(|name| { - Some((table_name.to_string(), name)) - }) - }) - } - - pub fn root_table_bound() -> (Vec, Vec) { - let op = |bound_id| { - format!( - "RootCatalog_{}", - bound_id, - ) - }; - - (op(BOUND_MIN_TAG).into_bytes(), op(BOUND_MAX_TAG).into_bytes()) + pub fn decode_root_table(bytes: &[u8]) -> Result { + Ok(String::from_utf8(bytes.to_vec())?) } } @@ -234,10 +244,11 @@ mod tests { use std::collections::BTreeSet; use std::ops::Bound; use std::sync::Arc; + use bytes::Bytes; use itertools::Itertools; use rust_decimal::Decimal; use crate::catalog::{ColumnCatalog, ColumnDesc, TableCatalog}; - use crate::storage::table_codec::{COLUMNS_ID_LEN, TableCodec}; + use crate::storage::table_codec::TableCodec; use crate::types::errors::TypeError; use crate::types::index::{Index, IndexMeta}; use crate::types::LogicalType; @@ -257,7 +268,7 @@ mod tests { ColumnDesc::new(LogicalType::Decimal(None,None), false, false) ), ]; - let table_catalog = TableCatalog::new(Arc::new("t1".to_string()), columns, vec![]).unwrap(); + let table_catalog = TableCatalog::new(Arc::new("t1".to_string()), columns).unwrap(); let codec = TableCodec { table: table_catalog.clone() }; (table_catalog, codec) } @@ -274,13 +285,8 @@ mod tests { Arc::new(DataValue::Decimal(Some(Decimal::new(1, 0)))), ] }; + let (_, bytes) = codec.encode_tuple(&tuple)?; - let (key, bytes) = codec.encode_tuple(&tuple)?; - - let mut test_key = format!("{}_Tuple_0_", table_catalog.name).into_bytes(); - tuple.id.clone().unwrap().to_primary_key(&mut test_key)?; - - assert_eq!(key.to_vec(), test_key); assert_eq!(codec.decode_tuple(&bytes), tuple); Ok(()) @@ -289,20 +295,11 @@ mod tests { #[test] fn test_root_catalog() { let (table_catalog, _) = build_table_codec(); - let (key, bytes) = TableCodec::encode_root_table(&table_catalog.name,2).unwrap(); - - assert_eq!( - String::from_utf8(key.to_vec()).ok().unwrap(), - format!( - "RootCatalog_0_{}", - table_catalog.name, - ) - ); + let (_, bytes) = TableCodec::encode_root_table(&table_catalog.name).unwrap(); - let (table_name, column_count) = TableCodec::decode_root_table(&key, &bytes).unwrap(); + let table_name = TableCodec::decode_root_table(&bytes).unwrap(); assert_eq!(table_name, table_catalog.name.as_str()); - assert_eq!(column_count, 2); } #[test] @@ -313,16 +310,8 @@ mod tests { name: "index_1".to_string(), is_unique: false, }; + let (_, bytes) = TableCodec::encode_index_meta(&"T1".to_string(), &index_meta)?; - let (key, bytes) = TableCodec::encode_index_meta(&"T1".to_string(), &index_meta)?; - - assert_eq!( - String::from_utf8(key.to_vec()).ok().unwrap(), - format!( - "T1_IndexMeta_0_{}", - index_meta.id - ) - ); assert_eq!(TableCodec::decode_index_meta(&bytes)?, index_meta); Ok(()) @@ -330,24 +319,15 @@ mod tests { #[test] fn test_table_codec_index() -> Result<(), TypeError> { - let (table_catalog, codec) = build_table_codec(); + let (_, codec) = build_table_codec(); let index = Index { id: 0, column_values: vec![Arc::new(DataValue::Int32(Some(0)))], }; let tuple_ids = vec![Arc::new(DataValue::Int32(Some(0)))]; + let (_, bytes) = codec.encode_index(&index, &tuple_ids)?; - let (key, bytes) = codec.encode_index(&index, &tuple_ids)?; - - let mut test_key = format!( - "{}_Index_0_{}_0", - table_catalog.name, - index.id, - ).into_bytes(); - index.column_values[0].to_index_key(&mut test_key)?; - - assert_eq!(key.to_vec(), test_key); assert_eq!(TableCodec::decode_index(&bytes)?, tuple_ids); Ok(()) @@ -357,20 +337,9 @@ mod tests { fn test_table_codec_column() { let (table_catalog, _) = build_table_codec(); let col = table_catalog.all_columns()[0].clone(); - let (key, bytes) = TableCodec::encode_column(&table_catalog.name, &col).unwrap(); - - assert_eq!( - String::from_utf8(key.to_vec()).ok().unwrap(), - format!( - "{}_Catalog_0_{}_{:0width$}", - table_catalog.name, - col.name, - col.id.unwrap(), - width = COLUMNS_ID_LEN - ) - ); + let (_, bytes) = TableCodec::encode_column(&col).unwrap(); - let (table_name, decode_col) = TableCodec::decode_column(&key, &bytes).unwrap(); + let (table_name, decode_col) = TableCodec::decode_column(&bytes).unwrap(); assert_eq!(&decode_col, col.as_ref()); assert_eq!(table_name, table_catalog.name); @@ -379,125 +348,171 @@ mod tests { #[test] fn test_table_codec_column_bound() { let mut set = BTreeSet::new(); - let op = |str: &str| { - str.to_string().into_bytes() + let op = |col_id: usize, table_name: &str| { + let mut col = ColumnCatalog::new( + "".to_string(), + false, + ColumnDesc { + column_datatype: LogicalType::Invalid, + is_primary: false, + is_unique: false, + } + ); + + col.table_name = Some(Arc::new(table_name.to_string())); + col.id = Some(col_id as u32); + + let (key, _) = TableCodec::encode_column(&col).unwrap(); + key }; - set.insert(op("T0_Catalog_0_C0_0")); - set.insert(op("T0_Catalog_0_C1_1")); - set.insert(op("T0_Catalog_0_C2_2")); + set.insert(op(0, "T0")); + set.insert(op(1, "T0")); + set.insert(op(2, "T0")); - set.insert(op("T1_Catalog_0_C0_0")); - set.insert(op("T1_Catalog_0_C1_1")); - set.insert(op("T1_Catalog_0_C2_2")); + set.insert(op(0, "T1")); + set.insert(op(1, "T1")); + set.insert(op(2, "T1")); - set.insert(op("T2_Catalog_0_C0_0")); - set.insert(op("T2_Catalog_0_C1_1")); - set.insert(op("T2_Catalog_0_C2_2")); + set.insert(op(0, "T2")); + set.insert(op(0, "T2")); + set.insert(op(0, "T2")); let (min, max) = TableCodec::columns_bound( &Arc::new("T1".to_string()) ); let vec = set - .range::, (Bound<&Vec>, Bound<&Vec>)>((Bound::Included(&min), Bound::Included(&max))) + .range::, Bound<&Bytes>)>(( + Bound::Included(&Bytes::from(min)), + Bound::Included(&Bytes::from(max)) + )) .collect_vec(); assert_eq!(vec.len(), 3); - assert_eq!(String::from_utf8(vec[0].clone()).unwrap(), "T1_Catalog_0_C0_0"); - assert_eq!(String::from_utf8(vec[1].clone()).unwrap(), "T1_Catalog_0_C1_1"); - assert_eq!(String::from_utf8(vec[2].clone()).unwrap(), "T1_Catalog_0_C2_2"); + assert_eq!(vec[0], &op(0, "T1")); + assert_eq!(vec[1], &op(1, "T1")); + assert_eq!(vec[2], &op(2, "T1")); } #[test] fn test_table_codec_index_meta_bound() { let mut set = BTreeSet::new(); - let op = |str: &str| { - str.to_string().into_bytes() + let op = |index_id: usize, table_name: &str| { + let index_meta = IndexMeta { + id: index_id as u32, + column_ids: vec![], + name: "".to_string(), + is_unique: false, + }; + + let (key, _) = TableCodec::encode_index_meta(&table_name.to_string(), &index_meta).unwrap(); + key }; - set.insert(op("T0_IndexMeta_0_0")); - set.insert(op("T0_IndexMeta_0_1")); - set.insert(op("T0_IndexMeta_0_2")); + set.insert(op(0, "T0")); + set.insert(op(1, "T0")); + set.insert(op(2, "T0")); - set.insert(op("T1_IndexMeta_0_0")); - set.insert(op("T1_IndexMeta_0_1")); - set.insert(op("T1_IndexMeta_0_2")); + set.insert(op(0, "T1")); + set.insert(op(1, "T1")); + set.insert(op(2, "T1")); - set.insert(op("T2_IndexMeta_0_0")); - set.insert(op("T2_IndexMeta_0_1")); - set.insert(op("T2_IndexMeta_0_2")); + set.insert(op(0, "T2")); + set.insert(op(1, "T2")); + set.insert(op(2, "T2")); let (min, max) = TableCodec::index_meta_bound(&"T1".to_string()); let vec = set - .range::, (Bound<&Vec>, Bound<&Vec>)>((Bound::Included(&min), Bound::Included(&max))) + .range::, Bound<&Bytes>)>(( + Bound::Included(&Bytes::from(min)), + Bound::Included(&Bytes::from(max)) + )) .collect_vec(); assert_eq!(vec.len(), 3); - assert_eq!(String::from_utf8(vec[0].clone()).unwrap(), "T1_IndexMeta_0_0"); - assert_eq!(String::from_utf8(vec[1].clone()).unwrap(), "T1_IndexMeta_0_1"); - assert_eq!(String::from_utf8(vec[2].clone()).unwrap(), "T1_IndexMeta_0_2"); + assert_eq!(vec[0], &op(0, "T1")); + assert_eq!(vec[1], &op(1, "T1")); + assert_eq!(vec[2], &op(2, "T1")); } #[test] fn test_table_codec_index_bound() { let mut set = BTreeSet::new(); - let op = |str: &str| { - str.to_string().into_bytes() + let table_codec = TableCodec { + table: TableCatalog::new(Arc::new("T0".to_string()), vec![]).unwrap(), }; - set.insert(op("T0_Index_0_0_0_0000000000000000000")); - set.insert(op("T0_Index_0_0_0_0000000000000000001")); - set.insert(op("T0_Index_0_0_0_0000000000000000002")); + let op = |value: DataValue, index_id: usize, table_codec: &TableCodec| { + let index = Index { + id: index_id as u32, + column_values: vec![Arc::new(value)], + }; - set.insert(op("T0_Index_0_1_0_0000000000000000000")); - set.insert(op("T0_Index_0_1_0_0000000000000000001")); - set.insert(op("T0_Index_0_1_0_0000000000000000002")); + table_codec.encode_index_key(&index).unwrap() + }; - set.insert(op("T0_Index_0_2_0_0000000000000000000")); - set.insert(op("T0_Index_0_2_0_0000000000000000001")); - set.insert(op("T0_Index_0_2_0_0000000000000000002")); + set.insert(op(DataValue::Int32(Some(0)), 0, &table_codec)); + set.insert(op(DataValue::Int32(Some(1)), 0, &table_codec)); + set.insert(op(DataValue::Int32(Some(2)), 0, &table_codec)); + + set.insert(op(DataValue::Int32(Some(0)), 1, &table_codec)); + set.insert(op(DataValue::Int32(Some(1)), 1, &table_codec)); + set.insert(op(DataValue::Int32(Some(2)), 1, &table_codec)); + + set.insert(op(DataValue::Int32(Some(0)), 2, &table_codec)); + set.insert(op(DataValue::Int32(Some(1)), 2, &table_codec)); + set.insert(op(DataValue::Int32(Some(2)), 2, &table_codec)); + + println!("{:#?}", set); - let table_codec = TableCodec { - table: TableCatalog::new(Arc::new("T0".to_string()), vec![], vec![]).unwrap(), - }; let (min, max) = table_codec.index_bound(&1); + println!("{:?}", min); + println!("{:?}", max); + let vec = set .range::, (Bound<&Vec>, Bound<&Vec>)>((Bound::Included(&min), Bound::Included(&max))) .collect_vec(); assert_eq!(vec.len(), 3); - assert_eq!(String::from_utf8(vec[0].clone()).unwrap(), "T0_Index_0_1_0_0000000000000000000"); - assert_eq!(String::from_utf8(vec[1].clone()).unwrap(), "T0_Index_0_1_0_0000000000000000001"); - assert_eq!(String::from_utf8(vec[2].clone()).unwrap(), "T0_Index_0_1_0_0000000000000000002"); + assert_eq!(vec[0], &op(DataValue::Int32(Some(0)), 1, &table_codec)); + assert_eq!(vec[1], &op(DataValue::Int32(Some(1)), 1, &table_codec)); + assert_eq!(vec[2], &op(DataValue::Int32(Some(2)), 1, &table_codec)); } #[test] fn test_table_codec_index_all_bound() { let mut set = BTreeSet::new(); - let op = |str: &str| { - str.to_string().into_bytes() + let op = |value: DataValue, index_id: usize, table_name: &str| { + let index = Index { + id: index_id as u32, + column_values: vec![Arc::new(value)], + }; + + TableCodec { + table: TableCatalog::new(Arc::new(table_name.to_string()), vec![]).unwrap() + }.encode_index_key(&index).unwrap() }; - set.insert(op("T0_Index_0_0_0_0000000000000000000")); - set.insert(op("T0_Index_0_0_0_0000000000000000001")); - set.insert(op("T0_Index_0_0_0_0000000000000000002")); + set.insert(op(DataValue::Int32(Some(0)), 0, "T0")); + set.insert(op(DataValue::Int32(Some(1)), 0, "T0")); + set.insert(op(DataValue::Int32(Some(2)), 0, "T0")); - set.insert(op("T1_Index_0_1_0_0000000000000000000")); - set.insert(op("T1_Index_0_1_0_0000000000000000001")); - set.insert(op("T1_Index_0_1_0_0000000000000000002")); + set.insert(op(DataValue::Int32(Some(0)), 0, "T1")); + set.insert(op(DataValue::Int32(Some(1)), 0, "T1")); + set.insert(op(DataValue::Int32(Some(2)), 0, "T1")); - set.insert(op("T2_Index_0_2_0_0000000000000000000")); - set.insert(op("T2_Index_0_2_0_0000000000000000001")); - set.insert(op("T2_Index_0_2_0_0000000000000000002")); + set.insert(op(DataValue::Int32(Some(0)), 0, "T2")); + set.insert(op(DataValue::Int32(Some(1)), 0, "T2")); + set.insert(op(DataValue::Int32(Some(2)), 0, "T2")); let table_codec = TableCodec { - table: TableCatalog::new(Arc::new("T1".to_string()), vec![], vec![]).unwrap(), + table: TableCatalog::new(Arc::new("T1".to_string()), vec![]).unwrap(), }; let (min, max) = table_codec.all_index_bound(); @@ -507,32 +522,34 @@ mod tests { assert_eq!(vec.len(), 3); - assert_eq!(String::from_utf8(vec[0].clone()).unwrap(), "T1_Index_0_1_0_0000000000000000000"); - assert_eq!(String::from_utf8(vec[1].clone()).unwrap(), "T1_Index_0_1_0_0000000000000000001"); - assert_eq!(String::from_utf8(vec[2].clone()).unwrap(), "T1_Index_0_1_0_0000000000000000002"); + assert_eq!(vec[0], &op(DataValue::Int32(Some(0)), 0, "T1")); + assert_eq!(vec[1], &op(DataValue::Int32(Some(1)), 0, "T1")); + assert_eq!(vec[2], &op(DataValue::Int32(Some(2)), 0, "T1")); } #[test] fn test_table_codec_tuple_bound() { let mut set = BTreeSet::new(); - let op = |str: &str| { - str.to_string().into_bytes() + let op = |tuple_id: DataValue, table_name: &str| { + TableCodec { + table: TableCatalog::new(Arc::new(table_name.to_string()), vec![]).unwrap() + }.encode_tuple_key(&Arc::new(tuple_id)).unwrap() }; - set.insert(op("T0_Tuple_0_0000000000000000000")); - set.insert(op("T0_Tuple_0_0000000000000000001")); - set.insert(op("T0_Tuple_0_0000000000000000002")); + set.insert(op(DataValue::Int32(Some(0)), "T0")); + set.insert(op(DataValue::Int32(Some(1)), "T0")); + set.insert(op(DataValue::Int32(Some(2)), "T0")); - set.insert(op("T1_Tuple_0_0000000000000000000")); - set.insert(op("T1_Tuple_0_0000000000000000001")); - set.insert(op("T1_Tuple_0_0000000000000000002")); + set.insert(op(DataValue::Int32(Some(0)), "T1")); + set.insert(op(DataValue::Int32(Some(1)), "T1")); + set.insert(op(DataValue::Int32(Some(2)), "T1")); - set.insert(op("T2_Tuple_0_0000000000000000000")); - set.insert(op("T2_Tuple_0_0000000000000000001")); - set.insert(op("T2_Tuple_0_0000000000000000002")); + set.insert(op(DataValue::Int32(Some(0)), "T2")); + set.insert(op(DataValue::Int32(Some(1)), "T2")); + set.insert(op(DataValue::Int32(Some(2)), "T2")); let table_codec = TableCodec { - table: TableCatalog::new(Arc::new("T1".to_string()), vec![], vec![]).unwrap(), + table: TableCatalog::new(Arc::new("T1".to_string()), vec![]).unwrap(), }; let (min, max) = table_codec.tuple_bound(); @@ -542,21 +559,25 @@ mod tests { assert_eq!(vec.len(), 3); - assert_eq!(String::from_utf8(vec[0].clone()).unwrap(), "T1_Tuple_0_0000000000000000000"); - assert_eq!(String::from_utf8(vec[1].clone()).unwrap(), "T1_Tuple_0_0000000000000000001"); - assert_eq!(String::from_utf8(vec[2].clone()).unwrap(), "T1_Tuple_0_0000000000000000002"); + assert_eq!(vec[0], &op(DataValue::Int32(Some(0)), "T1")); + assert_eq!(vec[1], &op(DataValue::Int32(Some(1)), "T1")); + assert_eq!(vec[2], &op(DataValue::Int32(Some(2)), "T1")); } #[test] fn test_root_codec_name_bound(){ let mut set = BTreeSet::new(); - let op = |str: &str| { - str.to_string().into_bytes() + let op = |table_name: &str| { + TableCodec::encode_root_table_key(&table_name.to_string()) }; - set.insert(op("RootCatalog_0_T0")); - set.insert(op("RootCatalog_0_T1")); - set.insert(op("RootCatalog_0_T2")); + set.insert(b"A".to_vec()); + + set.insert(op("T0")); + set.insert(op("T1")); + set.insert(op("T2")); + + set.insert(b"Z".to_vec()); let (min, max) = TableCodec::root_table_bound(); @@ -564,9 +585,9 @@ mod tests { .range::, (Bound<&Vec>, Bound<&Vec>)>((Bound::Included(&min), Bound::Included(&max))) .collect_vec(); - assert_eq!(String::from_utf8(vec[0].clone()).unwrap(), "RootCatalog_0_T0"); - assert_eq!(String::from_utf8(vec[1].clone()).unwrap(), "RootCatalog_0_T1"); - assert_eq!(String::from_utf8(vec[2].clone()).unwrap(), "RootCatalog_0_T2"); + assert_eq!(vec[0], &op("T0")); + assert_eq!(vec[1], &op("T1")); + assert_eq!(vec[2], &op("T2")); } } \ No newline at end of file diff --git a/src/types/errors.rs b/src/types/errors.rs index 701a664b..9ec307a1 100644 --- a/src/types/errors.rs +++ b/src/types/errors.rs @@ -1,5 +1,6 @@ use std::num::{ParseFloatError, ParseIntError, TryFromIntError}; use std::str::ParseBoolError; +use std::string::FromUtf8Error; use chrono::ParseError; #[derive(thiserror::Error, Debug)] @@ -58,4 +59,10 @@ pub enum TypeError { #[from] rust_decimal::Error, ), + #[error("from utf8")] + FromUtf8Error( + #[source] + #[from] + FromUtf8Error, + ) }