From 00d89589baaa8de70499bcd5b8fcaa450a2349be Mon Sep 17 00:00:00 2001 From: Noel Kwan <47273164+kwannoel@users.noreply.github.com> Date: Mon, 15 Apr 2024 23:10:20 +0800 Subject: [PATCH] fix(sqlsmith): ban failing functions and aggregations (#16295) --- ci/scripts/pr-fuzz-test.sh | 1 - ci/scripts/run-deterministic-fuzz-test.sh | 33 +++++++++++++++++++ ci/scripts/run-fuzz-test.sh | 9 ----- ci/workflows/pull-request.yml | 15 +++++++++ src/tests/sqlsmith/scripts/extract_queries.sh | 2 +- src/tests/sqlsmith/src/sql_gen/types.rs | 5 +++ 6 files changed, 54 insertions(+), 11 deletions(-) create mode 100755 ci/scripts/run-deterministic-fuzz-test.sh diff --git a/ci/scripts/pr-fuzz-test.sh b/ci/scripts/pr-fuzz-test.sh index 66923c4fb8a71..f33986e51e453 100755 --- a/ci/scripts/pr-fuzz-test.sh +++ b/ci/scripts/pr-fuzz-test.sh @@ -13,7 +13,6 @@ source ci/scripts/common.sh export RUN_SQLSMITH_FRONTEND=0 export RUN_SQLSMITH=1 export SQLSMITH_COUNT=100 -export TEST_NUM=32 echo "Enabled Sqlsmith tests." source ci/scripts/run-fuzz-test.sh diff --git a/ci/scripts/run-deterministic-fuzz-test.sh b/ci/scripts/run-deterministic-fuzz-test.sh new file mode 100755 index 0000000000000..c3b8cb2821fd4 --- /dev/null +++ b/ci/scripts/run-deterministic-fuzz-test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +export LOGDIR=.risingwave/log +export RUST_LOG=info +mkdir -p $LOGDIR + +while getopts 'p:' opt; do + case ${opt} in + p ) + profile=$OPTARG + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + ;; + esac +done +shift $((OPTIND -1)) + +source ci/scripts/common.sh + +echo "--- Download artifacts" +download-and-decompress-artifact risingwave_simulation . +chmod +x ./risingwave_simulation + +echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)" +seq 32 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log' \ No newline at end of file diff --git a/ci/scripts/run-fuzz-test.sh b/ci/scripts/run-fuzz-test.sh index 441b2797b6a4c..067c9b2e5e362 100755 --- a/ci/scripts/run-fuzz-test.sh +++ b/ci/scripts/run-fuzz-test.sh @@ -38,10 +38,6 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then download_and_prepare_rw "$profile" common - # echo "--- Download artifacts" - # download-and-decompress-artifact risingwave_simulation . - # chmod +x ./risingwave_simulation - echo "--- Download sqlsmith e2e bin" download-and-decompress-artifact sqlsmith-"$profile" target/debug/ mv target/debug/sqlsmith-"$profile" target/debug/sqlsmith @@ -70,9 +66,4 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then # Use that to reproduce logs on local machine. echo "--- Kill cluster" risedev kill - - # NOTE(Noel): This is disabled because once it fails, it keeps failing. - # That blocks PRs from getting through. - # echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)" - # seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log' fi diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index e0af290a42b09..c19b9a774fc12 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -754,6 +754,21 @@ steps: timeout_in_minutes: 15 retry: *auto-retry + - label: "deterministic fuzz test" + command: "ci/scripts/run-deterministic-fuzz-test.sh -p ci-dev" + if: build.pull_request.labels includes "ci/run-deterministic-sqlsmith-fuzzing-tests" || build.env("CI_STEPS") =~ /(^|,)deterministic-sqlsmith-fuzzing-tests?(,|$$)/ + depends_on: + - "build-simulation" + plugins: + - ./ci/plugins/swapfile + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 15 + retry: *auto-retry + - label: "enable ci/skip-ci only in draft PRs" if: build.pull_request.labels includes "ci/skip-ci" && !build.pull_request.draft commands: diff --git a/src/tests/sqlsmith/scripts/extract_queries.sh b/src/tests/sqlsmith/scripts/extract_queries.sh index 9abac600296aa..0dd4c6e720045 100755 --- a/src/tests/sqlsmith/scripts/extract_queries.sh +++ b/src/tests/sqlsmith/scripts/extract_queries.sh @@ -14,7 +14,7 @@ OUTPUT_FILE="$2" SHRUNK_OUTPUT_FILE="$2".shrunk echo "--- Extracting queries" -cat "$LOG_FILE" | rg "\[EXECUTING .*\]" | sed 's/.*\[EXECUTING .*\]: //' | sed 's/$/;/' > "$OUTPUT_FILE" +cat "$LOG_FILE" | rg "(\[EXECUTING|\[TEST)" | sed 's/.*\[EXECUTING .*\]: //' | sed 's/.*\[TEST .*\]: //' | sed 's/$/;/' > "$OUTPUT_FILE" echo "--- Extracted queries to $OUTPUT_FILE" echo "--- Shrinking queries" diff --git a/src/tests/sqlsmith/src/sql_gen/types.rs b/src/tests/sqlsmith/src/sql_gen/types.rs index b4f9d01d294a9..4591a91c83836 100644 --- a/src/tests/sqlsmith/src/sql_gen/types.rs +++ b/src/tests/sqlsmith/src/sql_gen/types.rs @@ -112,6 +112,10 @@ static FUNC_BAN_LIST: LazyLock> = LazyLock::new(|| { ExprType::Repeat, // The format argument needs to be handled specially. It is still generated in `gen_special_func`. ExprType::Decode, + // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293 + ExprType::Sqrt, + // ENABLE: https://github.com/risingwavelabs/risingwave/issues/16293 + ExprType::Pow, ] .into_iter() .collect() @@ -181,6 +185,7 @@ pub(crate) static AGG_FUNC_TABLE: LazyLock