diff --git a/src/tests/sqlsmith/src/runner.rs b/src/tests/sqlsmith/src/runner.rs index d008e72091e2c..c1d7a55515621 100644 --- a/src/tests/sqlsmith/src/runner.rs +++ b/src/tests/sqlsmith/src/runner.rs @@ -62,8 +62,9 @@ pub async fn test_sqlsmith( setup_sql: &str, ) { // Test percentage of skipped queries <=5% of sample size. + let threshold = 0.20; // permit at most 20% of queries to be skipped. let mut batch_skipped = 0; - let batch_sample_size = 100; + let batch_sample_size = 50; client .query("SET query_mode TO distributed;", &[]) .await @@ -75,12 +76,16 @@ pub async fn test_sqlsmith( batch_skipped += validate_response_with_skip_count(setup_sql, &format!("{};", sql), response); } - if batch_skipped > 5 { - panic!("skipped >5% batch queries"); + let skipped_percentage = batch_skipped as f64 / batch_sample_size as f64; + if skipped_percentage > threshold { + panic!( + "percentage of skipped batch queries = {}, threshold: {}", + skipped_percentage, threshold + ); } let mut stream_skipped = 0; - let stream_sample_size = 100; + let stream_sample_size = 50; for _ in 0..stream_sample_size { let (sql, table) = mview_sql_gen(rng, tables.clone(), "stream_query"); tracing::info!("Executing: {}", sql); @@ -90,8 +95,12 @@ pub async fn test_sqlsmith( drop_mview_table(&table, client).await; } - if stream_skipped > 5 { - panic!("skipped >5% stream queries"); + let skipped_percentage = stream_skipped as f64 / stream_sample_size as f64; + if skipped_percentage > threshold { + panic!( + "percentage of skipped batch queries = {}, threshold: {}", + skipped_percentage, threshold + ); } } @@ -129,6 +138,7 @@ async fn create_tables( for i in 0..10 { let (create_sql, table) = mview_sql_gen(rng, tables.clone(), &format!("m{}", i)); setup_sql.push_str(&format!("{};", &create_sql)); + tracing::info!("Executing MView Setup: {}", &create_sql); client.execute(&create_sql, &[]).await.unwrap(); tables.push(table.clone()); mviews.push(table); @@ -182,6 +192,13 @@ fn is_broken_chan_err(db_error: &DbError) -> bool { .contains("internal error: broken fifo_channel") } +/// Skip queries with unimplemented features +fn is_unimplemented_error(db_error: &DbError) -> bool { + db_error + .message() + .contains("Feature is not yet implemented") +} + /// Certain errors are permitted to occur. This is because: /// 1. It is more complex to generate queries without these errors. /// 2. These errors seldom occur, skipping them won't affect overall effectiveness of sqlsmith. @@ -190,6 +207,7 @@ fn is_permissible_error(db_error: &DbError) -> bool { (is_internal_error && is_broken_chan_err(db_error)) || is_numeric_out_of_range_err(db_error) || is_division_by_zero_err(db_error) + || is_unimplemented_error(db_error) } /// Validate client responses