feat: support Octets for Char/Varchar

Cargo.toml

@@ -58,7 +58,7 @@ regex                 = { version = "1.10.3" }
 rust_decimal          = { version = "1.34.3" }
 serde                 = { version = "1.0.197", features = ["derive", "rc"] }
 siphasher             = { version = "1.0.0", features = ["serde"] }
-sqlparser             = { version = "0.34.0" }
+sqlparser             = { version = "0.34.0", features = ["serde"] }
 strum_macros          = { version = "0.26.2" }
 thiserror             = { version = "1.0.58" }
 tokio                 = { version = "1.36.0", features = ["full"] }

README.md

@@ -17,6 +17,7 @@ Built by @KipData
 </h3>
 
 <p align="center">
+    <a href="https://summer-ospp.ac.cn/org/orgdetail/0b09d23d-2510-4537-aa9d-45158bb6bdc2"><img src="https://img.shields.io/badge/OSPP-KipData-3DA639?logo=opensourceinitiative"></a>
     &nbsp;
     <a href="https://github.com/KipData/KipSQL/actions/workflows/ci.yml"><img src="https://github.com/KipData/KipSQL/actions/workflows/ci.yml/badge.svg" alt="CI"></img></a>
     &nbsp;

src/bin/server.rs

@@ -198,7 +198,7 @@ fn encode_tuples<'a>(schema: &Schema, tuples: Vec<Tuple>) -> PgWireResult<QueryR
                 LogicalType::UBigint => encoder.encode_field(&value.u64().map(|v| v as i64)),
                 LogicalType::Float => encoder.encode_field(&value.float()),
                 LogicalType::Double => encoder.encode_field(&value.double()),
-                LogicalType::Char(_) | LogicalType::Varchar(_) => {
+                LogicalType::Char(..) | LogicalType::Varchar(..) => {
                     encoder.encode_field(&value.utf8())
                 }
                 LogicalType::Date => encoder.encode_field(&value.date()),
@@ -225,9 +225,9 @@ fn into_pg_type(data_type: &LogicalType) -> PgWireResult<Type> {
         LogicalType::Bigint | LogicalType::UBigint => Type::INT8,
         LogicalType::Float => Type::FLOAT4,
         LogicalType::Double => Type::FLOAT8,
-        LogicalType::Varchar(_) => Type::VARCHAR,
+        LogicalType::Varchar(..) => Type::VARCHAR,
         LogicalType::Date | LogicalType::DateTime => Type::DATE,
-        LogicalType::Char(_) => Type::CHAR,
+        LogicalType::Char(..) => Type::CHAR,
         LogicalType::Time => Type::TIME,
         LogicalType::Decimal(_, _) => todo!(),
         _ => {

src/binder/create_table.rs

@@ -146,6 +146,7 @@ mod tests {
     use crate::storage::kip::KipStorage;
     use crate::storage::Storage;
     use crate::types::LogicalType;
+    use sqlparser::ast::CharLengthUnits;
     use std::sync::atomic::AtomicUsize;
     use tempfile::TempDir;
 
@@ -177,7 +178,12 @@ mod tests {
                 assert_eq!(op.columns[1].nullable, true);
                 assert_eq!(
                     op.columns[1].desc,
-                    ColumnDesc::new(LogicalType::Varchar(Some(10)), false, false, None)
+                    ColumnDesc::new(
+                        LogicalType::Varchar(Some(10), CharLengthUnits::Characters),
+                        false,
+                        false,
+                        None
+                    )
                 );
             }
             _ => unreachable!(),

src/binder/expr.rs

@@ -4,8 +4,8 @@ use crate::expression;
 use crate::expression::agg::AggKind;
 use itertools::Itertools;
 use sqlparser::ast::{
-    BinaryOperator, DataType, Expr, Function, FunctionArg, FunctionArgExpr, Ident, Query,
-    UnaryOperator,
+    BinaryOperator, CharLengthUnits, DataType, Expr, Function, FunctionArg, FunctionArgExpr, Ident,
+    Query, UnaryOperator,
 };
 use std::slice;
 use std::sync::Arc;
@@ -69,7 +69,8 @@ impl<'a, T: Transaction> Binder<'a, T> {
                 let logical_type = LogicalType::try_from(data_type.clone())?;
                 let value = DataValue::Utf8 {
                     value: Some(value.to_string()),
-                    ty: Utf8Type::Variable,
+                    ty: Utf8Type::Variable(None),
+                    unit: CharLengthUnits::Characters,
                 }
                 .cast(&logical_type)?;
 
@@ -354,7 +355,7 @@ impl<'a, T: Transaction> Binder<'a, T> {
             | BinaryOperator::And
             | BinaryOperator::Or
             | BinaryOperator::Xor => LogicalType::Boolean,
-            BinaryOperator::StringConcat => LogicalType::Varchar(None),
+            BinaryOperator::StringConcat => LogicalType::Varchar(None, CharLengthUnits::Characters),
             _ => todo!(),
         };
 
@@ -603,7 +604,8 @@ impl<'a, T: Transaction> Binder<'a, T> {
     fn wildcard_expr() -> ScalarExpression {
         ScalarExpression::Constant(Arc::new(DataValue::Utf8 {
             value: Some("*".to_string()),
-            ty: Utf8Type::Variable,
+            ty: Utf8Type::Variable(None),
+            unit: CharLengthUnits::Characters,
         }))
     }
 }

src/catalog/column.rs

@@ -2,6 +2,7 @@ use crate::catalog::TableName;
 use crate::errors::DatabaseError;
 use crate::expression::ScalarExpression;
 use serde::{Deserialize, Serialize};
+use sqlparser::ast::CharLengthUnits;
 use std::hash::Hash;
 use std::sync::Arc;
 
@@ -50,7 +51,12 @@ impl ColumnCatalog {
                 table_name: None,
             },
             nullable: true,
-            desc: ColumnDesc::new(LogicalType::Varchar(None), false, false, None),
+            desc: ColumnDesc::new(
+                LogicalType::Varchar(None, CharLengthUnits::Characters),
+                false,
+                false,
+                None,
+            ),
         }
     }
 

src/execution/volcano/dml/analyze.rs

@@ -12,6 +12,7 @@ use crate::types::tuple::Tuple;
 use crate::types::value::{DataValue, Utf8Type};
 use futures_async_stream::try_stream;
 use itertools::Itertools;
+use sqlparser::ast::CharLengthUnits;
 use std::fmt::Formatter;
 use std::sync::Arc;
 use std::time::{SystemTime, UNIX_EPOCH};
@@ -108,7 +109,8 @@ impl Analyze {
             meta.to_file(&path)?;
             values.push(Arc::new(DataValue::Utf8 {
                 value: Some(path.clone()),
-                ty: Utf8Type::Variable,
+                ty: Utf8Type::Variable(None),
+                unit: CharLengthUnits::Characters,
             }));
             transaction.save_table_meta(&table_name, path, meta)?;
         }

src/execution/volcano/dml/copy_from_file.rs

@@ -105,6 +105,7 @@ mod tests {
     use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnSummary};
     use crate::db::DataBaseBuilder;
     use futures::StreamExt;
+    use sqlparser::ast::CharLengthUnits;
     use std::io::Write;
     use std::sync::Arc;
     use tempfile::TempDir;
@@ -148,7 +149,12 @@ mod tests {
                     table_name: None,
                 },
                 nullable: false,
-                desc: ColumnDesc::new(LogicalType::Varchar(Some(10)), false, false, None),
+                desc: ColumnDesc::new(
+                    LogicalType::Varchar(Some(10), CharLengthUnits::Characters),
+                    false,
+                    false,
+                    None,
+                ),
             }),
         ];
 

src/execution/volcano/dql/describe.rs

@@ -7,20 +7,24 @@ use crate::types::tuple::Tuple;
 use crate::types::value::{DataValue, Utf8Type, ValueRef};
 use futures_async_stream::try_stream;
 use lazy_static::lazy_static;
+use sqlparser::ast::CharLengthUnits;
 use std::sync::Arc;
 
 lazy_static! {
     static ref PRIMARY_KEY_TYPE: ValueRef = Arc::new(DataValue::Utf8 {
         value: Some(String::from("PRIMARY")),
-        ty: Utf8Type::Variable
+        ty: Utf8Type::Variable(None),
+        unit: CharLengthUnits::Characters
     });
     static ref UNIQUE_KEY_TYPE: ValueRef = Arc::new(DataValue::Utf8 {
         value: Some(String::from("UNIQUE")),
-        ty: Utf8Type::Variable
+        ty: Utf8Type::Variable(None),
+        unit: CharLengthUnits::Characters
     });
     static ref EMPTY_KEY_TYPE: ValueRef = Arc::new(DataValue::Utf8 {
         value: Some(String::from("EMPTY")),
-        ty: Utf8Type::Variable
+        ty: Utf8Type::Variable(None),
+        unit: CharLengthUnits::Characters
     });
 }
 
@@ -69,24 +73,29 @@ impl Describe {
             let values = vec![
                 Arc::new(DataValue::Utf8 {
                     value: Some(column.name().to_string()),
-                    ty: Utf8Type::Variable,
+                    ty: Utf8Type::Variable(None),
+                    unit: CharLengthUnits::Characters,
                 }),
                 Arc::new(DataValue::Utf8 {
                     value: Some(datatype.to_string()),
-                    ty: Utf8Type::Variable,
+                    ty: Utf8Type::Variable(None),
+                    unit: CharLengthUnits::Characters,
                 }),
                 Arc::new(DataValue::Utf8 {
                     value: datatype.raw_len().map(|len| len.to_string()),
-                    ty: Utf8Type::Variable,
+                    ty: Utf8Type::Variable(None),
+                    unit: CharLengthUnits::Characters,
                 }),
                 Arc::new(DataValue::Utf8 {
                     value: Some(column.nullable.to_string()),
-                    ty: Utf8Type::Variable,
+                    ty: Utf8Type::Variable(None),
+                    unit: CharLengthUnits::Characters,
                 }),
                 key_fn(column),
                 Arc::new(DataValue::Utf8 {
                     value: Some(default),
-                    ty: Utf8Type::Variable,
+                    ty: Utf8Type::Variable(None),
+                    unit: CharLengthUnits::Characters,
                 }),
             ];
             yield Tuple { id: None, values };

src/execution/volcano/dql/explain.rs

@@ -5,6 +5,7 @@ use crate::storage::Transaction;
 use crate::types::tuple::Tuple;
 use crate::types::value::{DataValue, Utf8Type};
 use futures_async_stream::try_stream;
+use sqlparser::ast::CharLengthUnits;
 use std::sync::Arc;
 
 pub struct Explain {
@@ -28,7 +29,8 @@ impl Explain {
     pub async fn _execute(self) {
         let values = vec![Arc::new(DataValue::Utf8 {
             value: Some(self.plan.explain(0)),
-            ty: Utf8Type::Variable,
+            ty: Utf8Type::Variable(None),
+            unit: CharLengthUnits::Characters,
         })];
 
         yield Tuple { id: None, values };

src/execution/volcano/dql/show_table.rs

@@ -5,6 +5,7 @@ use crate::storage::Transaction;
 use crate::types::tuple::Tuple;
 use crate::types::value::{DataValue, Utf8Type};
 use futures_async_stream::try_stream;
+use sqlparser::ast::CharLengthUnits;
 use std::sync::Arc;
 
 pub struct ShowTables;
@@ -23,7 +24,8 @@ impl ShowTables {
         for TableMeta { table_name } in metas {
             let values = vec![Arc::new(DataValue::Utf8 {
                 value: Some(table_name.to_string()),
-                ty: Utf8Type::Variable,
+                ty: Utf8Type::Variable(None),
+                unit: CharLengthUnits::Characters,
             })];
 
             yield Tuple { id: None, values };

src/expression/evaluator.rs

@@ -7,6 +7,7 @@ use crate::types::value::{DataValue, Utf8Type, ValueRef};
 use crate::types::LogicalType;
 use itertools::Itertools;
 use lazy_static::lazy_static;
+use sqlparser::ast::CharLengthUnits;
 use std::cmp;
 use std::cmp::Ordering;
 use std::sync::Arc;
@@ -25,7 +26,8 @@ macro_rules! eval_to_num {
         } else {
             return Ok(Arc::new(DataValue::Utf8 {
                 value: None,
-                ty: Utf8Type::Variable,
+                ty: Utf8Type::Variable(None),
+                unit: CharLengthUnits::Characters,
             }));
         }
     };
@@ -156,7 +158,7 @@ impl ScalarExpression {
                 from_expr,
             } => {
                 if let Some(mut string) = DataValue::clone(expr.eval(tuple, schema)?.as_ref())
-                    .cast(&LogicalType::Varchar(None))?
+                    .cast(&LogicalType::Varchar(None, CharLengthUnits::Characters))?
                     .utf8()
                 {
                     if let Some(from_expr) = from_expr {
@@ -169,7 +171,8 @@ impl ScalarExpression {
                         if from > len_i {
                             return Ok(Arc::new(DataValue::Utf8 {
                                 value: None,
-                                ty: Utf8Type::Variable,
+                                ty: Utf8Type::Variable(None),
+                                unit: CharLengthUnits::Characters,
                             }));
                         }
                         string = string.split_off(from as usize);
@@ -182,19 +185,21 @@ impl ScalarExpression {
 
                     Ok(Arc::new(DataValue::Utf8 {
                         value: Some(string),
-                        ty: Utf8Type::Variable,
+                        ty: Utf8Type::Variable(None),
+                        unit: CharLengthUnits::Characters,
                     }))
                 } else {
                     Ok(Arc::new(DataValue::Utf8 {
                         value: None,
-                        ty: Utf8Type::Variable,
+                        ty: Utf8Type::Variable(None),
+                        unit: CharLengthUnits::Characters,
                     }))
                 }
             }
             ScalarExpression::Position { expr, in_expr } => {
                 let unpack = |expr: &ScalarExpression| -> Result<String, DatabaseError> {
                     Ok(DataValue::clone(expr.eval(tuple, schema)?.as_ref())
-                        .cast(&LogicalType::Varchar(None))?
+                        .cast(&LogicalType::Varchar(None, CharLengthUnits::Characters))?
                         .utf8()
                         .unwrap_or("".to_owned()))
                 };

src/expression/mod.rs

@@ -5,7 +5,9 @@ use std::hash::Hash;
 use std::sync::Arc;
 use std::{fmt, mem};
 
-use sqlparser::ast::{BinaryOperator as SqlBinaryOperator, UnaryOperator as SqlUnaryOperator};
+use sqlparser::ast::{
+    BinaryOperator as SqlBinaryOperator, CharLengthUnits, UnaryOperator as SqlUnaryOperator,
+};
 
 use self::agg::AggKind;
 use crate::catalog::{ColumnCatalog, ColumnDesc, ColumnRef};
@@ -394,7 +396,9 @@ impl ScalarExpression {
             ScalarExpression::IsNull { .. }
             | ScalarExpression::In { .. }
             | ScalarExpression::Between { .. } => LogicalType::Boolean,
-            ScalarExpression::SubString { .. } => LogicalType::Varchar(None),
+            ScalarExpression::SubString { .. } => {
+                LogicalType::Varchar(None, CharLengthUnits::Characters)
+            }
             ScalarExpression::Position { .. } => LogicalType::Integer,
             ScalarExpression::Alias { expr, .. } | ScalarExpression::Reference { expr, .. } => {
                 expr.return_type()