fix(ci): sqlsmith test filepath (#8183)

* fix filepath difference

* use absolute filepath + run deterministic fuzzing in cron

* fix

* update pr template for sqlsmith

* add up to 100 queries

* generate 100 queries instead

* fix ci scripts

---------

Co-authored-by: TennyZhuang <[email protected]>

.github/pull_request_template.md

@@ -20,7 +20,7 @@ Please explain **IN DETAIL** what the changes are in this PR and why they are ne
 
 - [ ] I have written necessary rustdoc comments
 - [ ] I have added necessary unit tests and integration tests
-- [ ] I have added fuzzing tests or opened an issue to track them. (Optional, recommended for new SQL features).
+- [ ] I have added fuzzing tests or opened an issue to track them. (Optional, recommended for new SQL features #7934).
 - [ ] I have demonstrated that backward compatibility is not broken by breaking changes and created issues to track deprecated features to be removed in the future. (Please refer to the issue)
 - [ ] All checks passed in `./risedev check` (or alias, `./risedev c`)
 

ci/scripts/cron-e2e-test.sh

@@ -7,5 +7,4 @@ source ci/scripts/common.env.sh
 export RUN_COMPACTION=1;
 export RUN_META_BACKUP=1;
 export RUN_DELETE_RANGE=1;
-export RUN_DETERMINISTIC_SQLSMITH=1;
 source ci/scripts/run-e2e-test.sh

ci/scripts/cron-fuzz-test.sh

@@ -6,4 +6,5 @@ set -euo pipefail
 source ci/scripts/common.env.sh
 export RUN_SQLSMITH=1
 export SQLSMITH_COUNT=1000
+export TEST_NUM=100
 source ci/scripts/run-fuzz-test.sh

ci/scripts/deterministic-e2e-test.sh

@@ -40,9 +40,4 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
 seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (pre-generated-queries)"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries src/tests/sqlsmith/tests/freeze/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
-
-if [[ "$RUN_DETERMINISTIC_SQLSMITH" -eq "1" ]]; then
-  echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)"
-  seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
-fi
+seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/freeze/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'

ci/scripts/pr-fuzz-test.sh

@@ -4,5 +4,30 @@
 set -euo pipefail
 
 source ci/scripts/common.env.sh
-source ci/scripts/pr.env.sh
+
+set +e
+# Set features, depending on our workflow
+# If sqlsmith files are modified, we run tests with sqlsmith enabled.
+MATCHES="ci/scripts/pr.env.sh\
+\|ci/scripts/cron-fuzz-test.sh\
+\|ci/scripts/pr-fuzz-test.sh\
+\|ci/scripts/run-fuzz-test.sh\
+\|src/tests/sqlsmith"
+CHANGED=$(git diff --name-only origin/main | grep "$MATCHES")
+set -e
+
+if [[ -n "$CHANGED" ]]; then
+    echo "origin/main SHA: $(git rev-parse origin/main)"
+    echo "Changes to Sqlsmith source files detected:"
+    echo "$CHANGED"
+
+    export RUN_SQLSMITH=1
+    export SQLSMITH_COUNT=100
+    export TEST_NUM=32
+    echo "Enabled Sqlsmith tests."
+else
+# Otherwise we use default.
+    export RUN_SQLSMITH=0
+fi
+
 source ci/scripts/run-fuzz-test.sh

ci/scripts/pr.env.sh

@@ -3,35 +3,9 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
-set +e
-# Set features, depending on our workflow
-# If sqlsmith files are modified, we run tests with sqlsmith enabled.
-MATCHES="ci/scripts/pr.env.sh\
-\|ci/scripts/pr-fuzz-test.sh\
-\|ci/scripts/run-e2e-test.sh\
-\|ci/scripts/run-fuzz-test.sh\
-\|src/tests/sqlsmith"
-CHANGED=$(git diff --name-only origin/main | grep "$MATCHES")
-set -e
-
 # Don't run e2e compaction test in PR build
 export RUN_COMPACTION=0;
 # Don't run meta store backup/recovery test
 export RUN_META_BACKUP=0;
 # Don't run delete-range random test
-export RUN_DELETE_RANGE=0;
-# Don't run deterministic e2e fuzzing (only run pre-gen)
-export RUN_DETERMINISTIC_SQLSMITH=0;
-
-if [[ -n "$CHANGED" ]]; then
-    echo "origin/main SHA: $(git rev-parse origin/main)";
-    echo "Changes to Sqlsmith source files detected:";
-    echo "$CHANGED";
-
-    export RUN_SQLSMITH=1;
-    export SQLSMITH_COUNT=100;
-    echo "Enabled Sqlsmith tests.";
-else
-# Otherwise we use default.
-    export RUN_SQLSMITH=0;
-fi
+export RUN_DELETE_RANGE=0;

ci/scripts/run-fuzz-test.sh

@@ -26,6 +26,8 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
     buildkite-agent artifact download risedev-dev-"$profile" target/debug/
     mv target/debug/risingwave-"$profile" target/debug/risingwave
     mv target/debug/risedev-dev-"$profile" target/debug/risedev-dev
+    buildkite-agent artifact download risingwave_simulation .
+    chmod +x ./risingwave_simulation
 
     echo "--- Adjust permission"
     chmod +x ./target/debug/risingwave
@@ -54,4 +56,7 @@ if [[ "$RUN_SQLSMITH" -eq "1" ]]; then
     # Use that to reproduce logs on local machine.
     echo "--- Kill cluster"
     cargo make kill
+
+    echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)"
+    seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
 fi

src/tests/sqlsmith/scripts/gen_queries.sh

@@ -2,7 +2,7 @@
 
 set -euxo pipefail
 
-export TEST_NUM=32
+export TEST_NUM=100
 export RW_HOME="../../../.."
 export LOGDIR=".risingwave/log"
 export TESTS_FOLDER="src/tests/sqlsmith/tests"

src/tests/sqlsmith/tests/freeze/1/ddl.sql

@@ -11,11 +11,13 @@ CREATE TABLE auction (id BIGINT, item_name CHARACTER VARYING, description CHARAC
 CREATE TABLE bid (auction BIGINT, bidder BIGINT, price BIGINT, channel CHARACTER VARYING, url CHARACTER VARYING, date_time TIMESTAMP, extra CHARACTER VARYING);
 CREATE TABLE alltypes1 (c1 BOOLEAN, c2 SMALLINT, c3 INT, c4 BIGINT, c5 REAL, c6 DOUBLE, c7 NUMERIC, c8 DATE, c9 CHARACTER VARYING, c10 TIME, c11 TIMESTAMP, c13 INTERVAL, c14 STRUCT<a INT>, c15 INT[], c16 CHARACTER VARYING[]);
 CREATE TABLE alltypes2 (c1 BOOLEAN, c2 SMALLINT, c3 INT, c4 BIGINT, c5 REAL, c6 DOUBLE, c7 NUMERIC, c8 DATE, c9 CHARACTER VARYING, c10 TIME, c11 TIMESTAMP, c13 INTERVAL, c14 STRUCT<a INT>, c15 INT[], c16 CHARACTER VARYING[]);
-CREATE MATERIALIZED VIEW m1 AS SELECT (TRIM(LEADING ('PCMqvE5FrE') FROM t_0.c9)) AS col_0, (ARRAY[(INT '981'), (INT '42'), (INT '738'), (INT '763')]) AS col_1, t_0.c2 AS col_2, (INTERVAL '3600') AS col_3 FROM alltypes1 AS t_0 JOIN region AS t_1 ON t_0.c9 = t_1.r_comment AND t_0.c1 GROUP BY t_0.c2, t_0.c8, t_0.c15, t_0.c9, t_0.c16, t_1.r_name, t_0.c5, t_0.c7, t_0.c6 HAVING true;
-CREATE MATERIALIZED VIEW m2 AS SELECT t_0.c1 AS col_0, t_0.c1 AS col_1, t_0.c1 AS col_2, true AS col_3 FROM alltypes2 AS t_0 GROUP BY t_0.c1 HAVING ((832) > ((100) + (SMALLINT '746')));
-CREATE MATERIALIZED VIEW m3 AS SELECT t_1.p_size AS col_0 FROM m1 AS t_0 LEFT JOIN part AS t_1 ON t_0.col_0 = t_1.p_type AND true GROUP BY t_1.p_type, t_0.col_3, t_1.p_size, t_1.p_brand, t_1.p_comment;
-CREATE MATERIALIZED VIEW m4 AS WITH with_0 AS (SELECT t_1.l_shipmode AS col_0, 'kZrMWYVcBV' AS col_1 FROM lineitem AS t_1 LEFT JOIN customer AS t_2 ON t_1.l_shipmode = t_2.c_address GROUP BY t_1.l_quantity, t_1.l_returnflag, t_2.c_comment, t_1.l_partkey, t_1.l_shipmode HAVING (avg((INTERVAL '-900581')) = TIME '02:05:33')) SELECT (802) AS col_0 FROM with_0;
-CREATE MATERIALIZED VIEW m5 AS SELECT (upper((TRIM(t_0.r_comment)))) AS col_0, (coalesce(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, t_0.r_regionkey)) AS col_1, t_0.r_comment AS col_2 FROM region AS t_0 RIGHT JOIN region AS t_1 ON t_0.r_comment = t_1.r_comment AND true WHERE true GROUP BY t_0.r_regionkey, t_0.r_comment;
-CREATE MATERIALIZED VIEW m6 AS SELECT '2qpwrIfYuD' AS col_0 FROM (SELECT (TIMESTAMP '2022-01-02 01:05:34') AS col_0, (TRIM(('nUPadiPg61'))) AS col_1, (t_0.seller & t_0.seller) AS col_2, t_0.date_time AS col_3 FROM auction AS t_0 GROUP BY t_0.id, t_0.seller, t_0.date_time, t_0.item_name) AS sq_1 WHERE true GROUP BY sq_1.col_3 HAVING CAST((INT '681') AS BOOLEAN);
-CREATE MATERIALIZED VIEW m7 AS WITH with_0 AS (SELECT (hop_1.id % (SMALLINT '512')) AS col_0 FROM hop(person, person.date_time, INTERVAL '86400', INTERVAL '5702400') AS hop_1 WHERE false GROUP BY hop_1.id) SELECT (584) AS col_0, ((REAL '613')) AS col_1 FROM with_0 WHERE true;
-CREATE MATERIALIZED VIEW m8 AS SELECT (OVERLAY(sq_2.col_2 PLACING sq_2.col_2 FROM sq_2.col_1)) AS col_0, sq_2.col_2 AS col_1, max((REAL '0')) AS col_2 FROM (SELECT (INT '417') AS col_0, t_1.c3 AS col_1, t_1.c9 AS col_2 FROM customer AS t_0 JOIN alltypes2 AS t_1 ON t_0.c_phone = t_1.c9 WHERE t_1.c1 GROUP BY t_0.c_comment, t_1.c11, t_1.c3, t_0.c_custkey, t_1.c2, t_1.c9, t_0.c_mktsegment, t_1.c8, t_1.c16, t_1.c5, t_0.c_address) AS sq_2 WHERE false GROUP BY sq_2.col_2, sq_2.col_1;
+CREATE MATERIALIZED VIEW m0 AS SELECT (sq_1.col_2 - CAST(true AS INT)) AS col_0, sq_1.col_1 AS col_1 FROM (SELECT ((((INT '296') - (SMALLINT '-32768')) | (INT '253')) / t_0.id) AS col_0, ((INT '-2147483648') + (t_0.initial_bid | ((- (SMALLINT '277')) # (INT '227')))) AS col_1, (773) AS col_2 FROM auction AS t_0 WHERE false GROUP BY t_0.initial_bid, t_0.description, t_0.item_name, t_0.date_time, t_0.id) AS sq_1 WHERE true GROUP BY sq_1.col_1, sq_1.col_2;
+CREATE MATERIALIZED VIEW m1 AS SELECT CAST(NULL AS STRUCT<a INTERVAL, b NUMERIC>) AS col_0, (TRIM(TRAILING t_0.p_type FROM ('BrvccI19br'))) AS col_1, (CAST(CAST(t_0.p_size AS BOOLEAN) AS INT) + DATE '2022-01-02') AS col_2, true AS col_3 FROM part AS t_0 LEFT JOIN lineitem AS t_1 ON t_0.p_mfgr = t_1.l_returnflag WHERE false GROUP BY t_1.l_linestatus, t_1.l_suppkey, t_1.l_returnflag, t_1.l_shipinstruct, t_0.p_type, t_1.l_comment, t_0.p_retailprice, t_0.p_size, t_0.p_container HAVING true;
+CREATE MATERIALIZED VIEW m2 AS SELECT (INT '0') AS col_0, tumble_0.c7 AS col_1, (414) AS col_2 FROM tumble(alltypes1, alltypes1.c11, INTERVAL '91') AS tumble_0 WHERE tumble_0.c1 GROUP BY tumble_0.c16, tumble_0.c14, tumble_0.c10, tumble_0.c5, tumble_0.c2, tumble_0.c7, tumble_0.c13;
+CREATE MATERIALIZED VIEW m3 AS SELECT tumble_0.c1 AS col_0, tumble_0.c1 AS col_1, tumble_0.c1 AS col_2, true AS col_3 FROM tumble(alltypes2, alltypes2.c11, INTERVAL '48') AS tumble_0 WHERE tumble_0.c1 GROUP BY tumble_0.c1 HAVING ((832) = ((SMALLINT '981') - (INT '0')));
+CREATE MATERIALIZED VIEW m4 AS SELECT t_2.bidder AS col_0 FROM bid AS t_2 WHERE true GROUP BY t_2.extra, t_2.price, t_2.bidder HAVING false;
+CREATE MATERIALIZED VIEW m5 AS WITH with_0 AS (SELECT ((coalesce(NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, sq_3.col_0)) & sq_3.col_0) AS col_0 FROM (WITH with_1 AS (SELECT (((INT '900') + DATE '2022-01-02') + (INTERVAL '-595892')) AS col_0, t_2.expires AS col_1, TIMESTAMP '2022-01-02 02:04:33' AS col_2 FROM auction AS t_2 WHERE (t_2.id = (FLOAT '275')) GROUP BY t_2.expires) SELECT (SMALLINT '487') AS col_0, (802) AS col_1 FROM with_1) AS sq_3 WHERE true GROUP BY sq_3.col_0 HAVING (false)) SELECT TIMESTAMP '2022-01-01 15:23:14' AS col_0, false AS col_1, (INT '518') AS col_2 FROM with_0;
+CREATE MATERIALIZED VIEW m6 AS SELECT t_2.col_2 AS col_0, t_2.col_2 AS col_1, ((BIGINT '687') - ((SMALLINT '-16176') / t_2.col_2)) AS col_2, t_2.col_2 AS col_3 FROM m2 AS t_2 GROUP BY t_2.col_2 HAVING true;
+CREATE MATERIALIZED VIEW m7 AS SELECT t_1.col_1 AS col_0, (substr(t_0.r_name, t_1.col_0)) AS col_1 FROM region AS t_0 RIGHT JOIN m2 AS t_1 ON t_0.r_regionkey = t_1.col_0 WHERE true GROUP BY t_1.col_1, t_1.col_0, t_0.r_name HAVING false;
+CREATE MATERIALIZED VIEW m8 AS WITH with_0 AS (SELECT hop_1.c15 AS col_0 FROM hop(alltypes2, alltypes2.c11, INTERVAL '1', INTERVAL '53') AS hop_1 GROUP BY hop_1.c9, hop_1.c6, hop_1.c5, hop_1.c4, hop_1.c15, hop_1.c11) SELECT ((418) - (BIGINT '876')) AS col_0, (- (REAL '0')) AS col_1 FROM with_0;
+CREATE MATERIALIZED VIEW m9 AS SELECT tumble_0.c15 AS col_0, (tumble_0.c5 > (BIGINT '6871269807316363457')) AS col_1 FROM tumble(alltypes2, alltypes2.c11, INTERVAL '5') AS tumble_0 WHERE tumble_0.c1 GROUP BY tumble_0.c2, tumble_0.c1, tumble_0.c15, tumble_0.c5, tumble_0.c14, tumble_0.c11, tumble_0.c8 HAVING tumble_0.c1;