From a0c1749ebb7fffcd8aeade0693ada14ae3b1af27 Mon Sep 17 00:00:00 2001 From: CookiePieWw <1035325592@qq.com> Date: Wed, 15 May 2024 03:02:37 +0800 Subject: [PATCH] fix: compatibility with mysql types --- tests-fuzz/Cargo.toml | 2 +- tests-fuzz/src/generator.rs | 6 +- tests-fuzz/src/generator/insert_expr.rs | 6 +- tests-fuzz/src/ir.rs | 76 +++++++++++++++++++ tests-fuzz/src/ir/insert_expr.rs | 2 +- tests-fuzz/src/validator/row.rs | 13 +++- tests-fuzz/targets/fuzz_insert.rs | 8 +- .../targets/fuzz_insert_logical_table.rs | 5 +- 8 files changed, 107 insertions(+), 11 deletions(-) diff --git a/tests-fuzz/Cargo.toml b/tests-fuzz/Cargo.toml index 075b54f6c41e..22a44e5cb094 100644 --- a/tests-fuzz/Cargo.toml +++ b/tests-fuzz/Cargo.toml @@ -17,13 +17,13 @@ unstable = ["nix"] [dependencies] arbitrary = { version = "1.3.0", features = ["derive"] } async-trait = { workspace = true } +chrono = { workspace = true } common-error = { workspace = true } common-macro = { workspace = true } common-query = { workspace = true } common-runtime = { workspace = true } common-telemetry = { workspace = true } common-time = { workspace = true } -chrono = { workspace = true } datatypes = { workspace = true } derive_builder = { workspace = true } dotenv = "0.15" diff --git a/tests-fuzz/src/generator.rs b/tests-fuzz/src/generator.rs index 2f9de0770c98..f4da5ae2353a 100644 --- a/tests-fuzz/src/generator.rs +++ b/tests-fuzz/src/generator.rs @@ -20,11 +20,12 @@ pub mod select_expr; use std::fmt; use datatypes::data_type::ConcreteDataType; +use datatypes::value::Value; use rand::Rng; use crate::error::Error; use crate::ir::create_expr::ColumnOption; -use crate::ir::{AlterTableExpr, CreateTableExpr}; +use crate::ir::{AlterTableExpr, CreateTableExpr, Ident}; pub type CreateTableExprGenerator = Box + Sync + Send>; @@ -36,6 +37,9 @@ pub type ColumnOptionGenerator = Box Vec pub type ConcreteDataTypeGenerator = Box>; +pub type ValueGenerator = + Box>) -> Value>; + pub trait Generator { type Error: Sync + Send + fmt::Debug; diff --git a/tests-fuzz/src/generator/insert_expr.rs b/tests-fuzz/src/generator/insert_expr.rs index db077a470f1c..2549e6bdd01a 100644 --- a/tests-fuzz/src/generator/insert_expr.rs +++ b/tests-fuzz/src/generator/insert_expr.rs @@ -22,7 +22,7 @@ use rand::Rng; use crate::context::TableContextRef; use crate::error::{Error, Result}; use crate::fake::WordGenerator; -use crate::generator::{Generator, Random}; +use crate::generator::{Generator, Random, ValueGenerator}; use crate::ir::insert_expr::{InsertIntoExpr, RowValue}; use crate::ir::{generate_random_value, Ident}; @@ -37,6 +37,8 @@ pub struct InsertExprGenerator { rows: usize, #[builder(default = "Box::new(WordGenerator)")] word_generator: Box>, + #[builder(default = "Box::new(generate_random_value)")] + value_generator: ValueGenerator, #[builder(default)] _phantom: PhantomData, } @@ -81,7 +83,7 @@ impl Generator for InsertExprGenerator { continue; } - row.push(RowValue::Value(generate_random_value( + row.push(RowValue::Value((self.value_generator)( rng, &column.column_type, Some(self.word_generator.as_ref()), diff --git a/tests-fuzz/src/ir.rs b/tests-fuzz/src/ir.rs index eb6ee105b2c8..39e9322e4c74 100644 --- a/tests-fuzz/src/ir.rs +++ b/tests-fuzz/src/ir.rs @@ -65,10 +65,21 @@ lazy_static! { ]; pub static ref STRING_DATA_TYPES: Vec = vec![ConcreteDataType::string_datatype()]; + pub static ref MYSQL_TS_DATA_TYPES: Vec = vec![ + // MySQL only permits fractional seconds with up to microseconds (6 digits) precision. + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_second_datatype(), + ]; } impl_random!(ConcreteDataType, ColumnTypeGenerator, DATA_TYPES); impl_random!(ConcreteDataType, TsColumnTypeGenerator, TS_DATA_TYPES); +impl_random!( + ConcreteDataType, + MySQLTsColumnTypeGenerator, + MYSQL_TS_DATA_TYPES +); impl_random!( ConcreteDataType, PartibleColumnTypeGenerator, @@ -82,6 +93,7 @@ impl_random!( pub struct ColumnTypeGenerator; pub struct TsColumnTypeGenerator; +pub struct MySQLTsColumnTypeGenerator; pub struct PartibleColumnTypeGenerator; pub struct StringColumnTypeGenerator; @@ -110,6 +122,31 @@ pub fn generate_random_value( } } +/// Generates a random [Value] for MySQL. +pub fn generate_random_value_for_mysql( + rng: &mut R, + datatype: &ConcreteDataType, + random_str: Option<&dyn Random>, +) -> Value { + match datatype { + &ConcreteDataType::Boolean(_) => Value::from(rng.gen::()), + ConcreteDataType::Int16(_) => Value::from(rng.gen::()), + ConcreteDataType::Int32(_) => Value::from(rng.gen::()), + ConcreteDataType::Int64(_) => Value::from(rng.gen::()), + ConcreteDataType::Float32(_) => Value::from(rng.gen::()), + ConcreteDataType::Float64(_) => Value::from(rng.gen::()), + ConcreteDataType::String(_) => match random_str { + Some(random) => Value::from(random.gen(rng).value), + None => Value::from(rng.gen::().to_string()), + }, + ConcreteDataType::Date(_) => generate_random_date(rng), + ConcreteDataType::DateTime(_) => generate_random_datetime(rng), + &ConcreteDataType::Timestamp(ts_type) => generate_random_timestamp_for_mysql(rng, ts_type), + + _ => unimplemented!("unsupported type: {datatype}"), + } +} + fn generate_random_timestamp(rng: &mut R, ts_type: TimestampType) -> Value { let v = match ts_type { TimestampType::Second(_) => { @@ -140,6 +177,37 @@ fn generate_random_timestamp(rng: &mut R, ts_type: TimestampType) -> Val Value::from(v) } +// MySQL supports timestamp from '1970-01-01 00:00:01.000000' to '2038-01-19 03:14:07.499999' +fn generate_random_timestamp_for_mysql(rng: &mut R, ts_type: TimestampType) -> Value { + let v = match ts_type { + TimestampType::Second(_) => { + let min = 1; + let max = 2_147_483_647; + let value = rng.gen_range(min..=max); + Timestamp::new_second(value) + } + TimestampType::Millisecond(_) => { + let min = 1000; + let max = 2_147_483_647_499; + let value = rng.gen_range(min..=max); + Timestamp::new_millisecond(value) + } + TimestampType::Microsecond(_) => { + let min = 1_000_000; + let max = 2_147_483_647_499_999; + let value = rng.gen_range(min..=max); + Timestamp::new_microsecond(value) + } + TimestampType::Nanosecond(_) => { + let min = 1_000_000_000; + let max = 2_147_483_647_499_999_000; + let value = rng.gen_range(min..=max); + Timestamp::new_nanosecond(value) + } + }; + Value::from(v) +} + fn generate_random_datetime(rng: &mut R) -> Value { let min = i64::from(Timestamp::MIN_MILLISECOND); let max = i64::from(Timestamp::MAX_MILLISECOND); @@ -258,6 +326,14 @@ impl Column { ) }) } + + // Returns default value if it has. + pub fn default_value(&self) -> Option<&Value> { + self.options.iter().find_map(|opt| match opt { + ColumnOption::DefaultValue(value) => Some(value), + _ => None, + }) + } } /// Returns droppable columns. i.e., non-primary key columns, non-ts columns. diff --git a/tests-fuzz/src/ir/insert_expr.rs b/tests-fuzz/src/ir/insert_expr.rs index e7a253b8970e..1b1c19537675 100644 --- a/tests-fuzz/src/ir/insert_expr.rs +++ b/tests-fuzz/src/ir/insert_expr.rs @@ -27,7 +27,7 @@ pub struct InsertIntoExpr { pub type RowValues = Vec; -#[derive(PartialEq, PartialOrd)] +#[derive(PartialEq, PartialOrd, Clone)] pub enum RowValue { Value(Value), Default, diff --git a/tests-fuzz/src/validator/row.rs b/tests-fuzz/src/validator/row.rs index 334e1d6b8234..44172991225c 100644 --- a/tests-fuzz/src/validator/row.rs +++ b/tests-fuzz/src/validator/row.rs @@ -27,6 +27,7 @@ use crate::ir::insert_expr::{RowValue, RowValues}; /// Asserts fetched_rows are equal to rows pub fn assert_eq<'a, DB>( + columns: &[crate::ir::Column], fetched_rows: &'a [::Row], rows: &[RowValues], ) -> Result<()> @@ -115,9 +116,17 @@ where } }; - // println!("Expected value: {:?}, got: {:?}", value, fetched_value); + let value = match value { + // In MySQL, boolean is stored as TINYINT(1) + RowValue::Value(Value::Boolean(v)) => RowValue::Value(Value::Int8(*v as i8)), + RowValue::Default => match columns[idx].default_value().unwrap().clone() { + Value::Boolean(v) => RowValue::Value(Value::Int8(v as i8)), + default_value => RowValue::Value(default_value), + }, + _ => value.clone(), + }; ensure!( - value == &fetched_value, + value == fetched_value, error::AssertSnafu { reason: format!("Expected value: {:?}, got: {:?}", value, fetched_value) } diff --git a/tests-fuzz/targets/fuzz_insert.rs b/tests-fuzz/targets/fuzz_insert.rs index 19212d9ebaff..eab40cb7ec9a 100644 --- a/tests-fuzz/targets/fuzz_insert.rs +++ b/tests-fuzz/targets/fuzz_insert.rs @@ -32,7 +32,9 @@ use tests_fuzz::fake::{ use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder; use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder; use tests_fuzz::generator::Generator; -use tests_fuzz::ir::{CreateTableExpr, InsertIntoExpr}; +use tests_fuzz::ir::{ + generate_random_value_for_mysql, CreateTableExpr, InsertIntoExpr, MySQLTsColumnTypeGenerator, +}; use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; use tests_fuzz::translator::mysql::insert_expr::InsertIntoExprTranslator; use tests_fuzz::translator::DslTranslator; @@ -81,6 +83,7 @@ fn generate_create_expr( ))) .columns(input.columns) .engine("mito") + .ts_column_type_generator(Box::new(MySQLTsColumnTypeGenerator)) .build() .unwrap(); create_table_generator.generate(rng) @@ -97,6 +100,7 @@ fn generate_insert_expr( .table_ctx(table_ctx) .omit_column_list(omit_column_list) .rows(input.rows) + .value_generator(Box::new(generate_random_value_for_mysql)) .build() .unwrap(); insert_generator.generate(rng) @@ -166,7 +170,7 @@ async fn execute_insert(ctx: FuzzContext, input: FuzzInput) -> Result<()> { .cmp(&b[ts_column_idx_in_insert]) .unwrap() }); - validator::row::assert_eq::(&fetched_rows, &expected_rows)?; + validator::row::assert_eq::(&insert_expr.columns, &fetched_rows, &expected_rows)?; // Cleans up let sql = format!("DROP TABLE {}", create_expr.table_name); diff --git a/tests-fuzz/targets/fuzz_insert_logical_table.rs b/tests-fuzz/targets/fuzz_insert_logical_table.rs index 8a8f4c5a53fa..fc8b2f9bd775 100644 --- a/tests-fuzz/targets/fuzz_insert_logical_table.rs +++ b/tests-fuzz/targets/fuzz_insert_logical_table.rs @@ -34,7 +34,7 @@ use tests_fuzz::generator::create_expr::{ }; use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder; use tests_fuzz::generator::Generator; -use tests_fuzz::ir::{CreateTableExpr, InsertIntoExpr}; +use tests_fuzz::ir::{generate_random_value_for_mysql, CreateTableExpr, InsertIntoExpr}; use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; use tests_fuzz::translator::mysql::insert_expr::InsertIntoExprTranslator; use tests_fuzz::translator::DslTranslator; @@ -108,6 +108,7 @@ fn generate_insert_expr( .omit_column_list(false) .table_ctx(table_ctx) .rows(input.rows) + .value_generator(Box::new(generate_random_value_for_mysql)) .build() .unwrap(); insert_generator.generate(rng) @@ -193,7 +194,7 @@ async fn execute_insert(ctx: FuzzContext, input: FuzzInput) -> Result<()> { .cmp(&b[ts_column_idx_in_insert]) .unwrap() }); - validator::row::assert_eq::(&fetched_rows, &expected_rows)?; + validator::row::assert_eq::(&insert_expr.columns, &fetched_rows, &expected_rows)?; // Clean up logical table let sql = format!("DROP TABLE {}", create_logical_table_expr.table_name);