fix(sqlsmith): ban failing functions and aggregations (#16295)

ci/scripts/pr-fuzz-test.sh

@@ -13,7 +13,6 @@ source ci/scripts/common.sh
 export RUN_SQLSMITH_FRONTEND=0
 export RUN_SQLSMITH=1
 export SQLSMITH_COUNT=100
-export TEST_NUM=32
 echo "Enabled Sqlsmith tests."
 
 source ci/scripts/run-fuzz-test.sh

ci/scripts/run-deterministic-fuzz-test.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+export LOGDIR=.risingwave/log
+export RUST_LOG=info
+mkdir -p $LOGDIR
+
+while getopts 'p:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+source ci/scripts/common.sh
+
+echo "--- Download artifacts"
+download-and-decompress-artifact risingwave_simulation .
+chmod +x ./risingwave_simulation
+
+echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)"
+seq 32 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'

ci/scripts/run-fuzz-test.sh

@@ -38,10 +38,6 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
 
     download_and_prepare_rw "$profile" common
 
-    #    echo "--- Download artifacts"
-    #    download-and-decompress-artifact risingwave_simulation .
-    #    chmod +x ./risingwave_simulation
-
     echo "--- Download sqlsmith e2e bin"
     download-and-decompress-artifact sqlsmith-"$profile" target/debug/
     mv target/debug/sqlsmith-"$profile" target/debug/sqlsmith
@@ -70,9 +66,4 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
     # Use that to reproduce logs on local machine.
     echo "--- Kill cluster"
     risedev kill
-
-    # NOTE(Noel): This is disabled because once it fails, it keeps failing.
-    # That blocks PRs from getting through.
-    # echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)"
-    # seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
 fi

ci/workflows/pull-request.yml

@@ -754,6 +754,21 @@ steps:
     timeout_in_minutes: 15
     retry: *auto-retry
 
+  - label: "deterministic fuzz test"
+    command: "ci/scripts/run-deterministic-fuzz-test.sh -p ci-dev"
+    if: build.pull_request.labels includes "ci/run-deterministic-sqlsmith-fuzzing-tests" || build.env("CI_STEPS") =~ /(^|,)deterministic-sqlsmith-fuzzing-tests?(,|$$)/
+    depends_on:
+      - "build-simulation"
+    plugins:
+      - ./ci/plugins/swapfile
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 15
+    retry: *auto-retry
+
   - label: "enable ci/skip-ci only in draft PRs"
     if: build.pull_request.labels includes "ci/skip-ci" && !build.pull_request.draft
     commands:

src/tests/sqlsmith/scripts/extract_queries.sh

@@ -14,7 +14,7 @@ OUTPUT_FILE="$2"
 SHRUNK_OUTPUT_FILE="$2".shrunk
 
 echo "--- Extracting queries"
-cat "$LOG_FILE" | rg "\[EXECUTING .*\]" | sed 's/.*\[EXECUTING .*\]: //' | sed 's/$/;/' > "$OUTPUT_FILE"
+cat "$LOG_FILE" | rg "(\[EXECUTING|\[TEST)" | sed 's/.*\[EXECUTING .*\]: //' | sed 's/.*\[TEST .*\]: //' | sed 's/$/;/' > "$OUTPUT_FILE"
 echo "--- Extracted queries to $OUTPUT_FILE"
 
 echo "--- Shrinking queries"

src/tests/sqlsmith/src/sql_gen/types.rs

@@ -112,6 +112,10 @@ static FUNC_BAN_LIST: LazyLock<HashSet<ExprType>> = LazyLock::new(|| {
         ExprType::Repeat,
         // The format argument needs to be handled specially. It is still generated in `gen_special_func`.
         ExprType::Decode,
+        // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
+        ExprType::Sqrt,
+        // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
+        ExprType::Pow,
     ]
     .into_iter()
     .collect()
@@ -181,6 +185,7 @@ pub(crate) static AGG_FUNC_TABLE: LazyLock<HashMap<DataType, Vec<&'static FuncSi
                         AggKind::PercentileCont,
                         AggKind::PercentileDisc,
                         AggKind::Mode,
+                        AggKind::JsonbObjectAgg, // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293
                     ]
                     .contains(&func.name.as_aggregate())
                     // Exclude 2 phase agg global sum.