feat(sqlsmith): ignore errors from unimplemented features (risingwave…

…labs#6603)

* skip unimplemented features

* fmt

* typo

* trace setup sql commands

* rollback blockers

* remove redundant trace

* avoid timeout

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

src/tests/sqlsmith/src/runner.rs

@@ -62,8 +62,9 @@ pub async fn test_sqlsmith<R: Rng>(
     setup_sql: &str,
 ) {
     // Test percentage of skipped queries <=5% of sample size.
+    let threshold = 0.20; // permit at most 20% of queries to be skipped.
     let mut batch_skipped = 0;
-    let batch_sample_size = 100;
+    let batch_sample_size = 50;
     client
         .query("SET query_mode TO distributed;", &[])
         .await
@@ -75,12 +76,16 @@ pub async fn test_sqlsmith<R: Rng>(
         batch_skipped +=
             validate_response_with_skip_count(setup_sql, &format!("{};", sql), response);
     }
-    if batch_skipped > 5 {
-        panic!("skipped >5% batch queries");
+    let skipped_percentage = batch_skipped as f64 / batch_sample_size as f64;
+    if skipped_percentage > threshold {
+        panic!(
+            "percentage of skipped batch queries = {}, threshold: {}",
+            skipped_percentage, threshold
+        );
     }
 
     let mut stream_skipped = 0;
-    let stream_sample_size = 100;
+    let stream_sample_size = 50;
     for _ in 0..stream_sample_size {
         let (sql, table) = mview_sql_gen(rng, tables.clone(), "stream_query");
         tracing::info!("Executing: {}", sql);
@@ -90,8 +95,12 @@ pub async fn test_sqlsmith<R: Rng>(
         drop_mview_table(&table, client).await;
     }
 
-    if stream_skipped > 5 {
-        panic!("skipped >5% stream queries");
+    let skipped_percentage = stream_skipped as f64 / stream_sample_size as f64;
+    if skipped_percentage > threshold {
+        panic!(
+            "percentage of skipped batch queries = {}, threshold: {}",
+            skipped_percentage, threshold
+        );
     }
 }
 
@@ -129,6 +138,7 @@ async fn create_tables(
     for i in 0..10 {
         let (create_sql, table) = mview_sql_gen(rng, tables.clone(), &format!("m{}", i));
         setup_sql.push_str(&format!("{};", &create_sql));
+        tracing::info!("Executing MView Setup: {}", &create_sql);
         client.execute(&create_sql, &[]).await.unwrap();
         tables.push(table.clone());
         mviews.push(table);
@@ -182,6 +192,13 @@ fn is_broken_chan_err(db_error: &DbError) -> bool {
         .contains("internal error: broken fifo_channel")
 }
 
+/// Skip queries with unimplemented features
+fn is_unimplemented_error(db_error: &DbError) -> bool {
+    db_error
+        .message()
+        .contains("Feature is not yet implemented")
+}
+
 /// Certain errors are permitted to occur. This is because:
 /// 1. It is more complex to generate queries without these errors.
 /// 2. These errors seldom occur, skipping them won't affect overall effectiveness of sqlsmith.
@@ -190,6 +207,7 @@ fn is_permissible_error(db_error: &DbError) -> bool {
     (is_internal_error && is_broken_chan_err(db_error))
         || is_numeric_out_of_range_err(db_error)
         || is_division_by_zero_err(db_error)
+        || is_unimplemented_error(db_error)
 }
 
 /// Validate client responses