feat(test): switch meta to sql backend in simluate test (#15612)

Signed-off-by: Kevin Axel <[email protected]> Co-authored-by: Noel Kwan <[email protected]> Co-authored-by: Noel Kwan <[email protected]>
risingwavelabs · Aug 19, 2024 · d5e832c · d5e832c
1 parent 11f0acf
commit d5e832c
Show file tree

Hide file tree

Showing 9 changed files with 182 additions and 86 deletions.
diff --git a/.gitignore b/.gitignore
@@ -80,6 +80,9 @@ e2e_test/iceberg/spark-*-bin*
 
 **/poetry.lock
 
+**/*.sqlite
+**/*.sqlite-journal
+
 *.slt.temp
 
 .direnv/

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -193,7 +193,7 @@ sea-orm = { version = "0.12.14", features = [
     "sqlx-sqlite",
     "runtime-tokio-native-tls",
 ] }
-sqlx = "0.7"
+sqlx = { version = "0.7.3", default-features = false, features = ["bigdecimal", "chrono", "json", "mysql", "postgres", "runtime-tokio-native-tls", "rust_decimal", "sqlite", "time", "uuid"] }
 tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "0dd1055", features = ["net", "fs"] }
 tokio-util = "0.7"
 tracing-opentelemetry = "0.24"
@@ -340,6 +340,10 @@ getrandom = { git = "https://github.com/madsim-rs/getrandom.git", rev = "e79a7ae
 # tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "0dd1055" }
 tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = "95e2fd3" }
 tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88" }
+# NOTE(kwannoel): patch sqlx to use a version compatible with madsim.
+# If we can merge in: https://github.com/madsim-rs/sqlx/pull/2,
+# we can change it to patch madsim version instead.
+sqlx = { git = "https://github.com/kwannoel/sqlx.git", rev = "5a32ad8ce842176c202df30910e1416655eda071" }
 futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" }
 # patch to remove preserve_order from serde_json
 bson = { git = "https://github.com/risingwavelabs/bson-rust", rev = "e5175ec" }

diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh
@@ -19,7 +19,8 @@ risingwave_meta::barrier::progress=debug"
 
 # Extra logs you can enable if the existing trace does not give enough info.
 #risingwave_stream::executor::backfill=trace,
-#risingwave_meta::barrier::progress=debug,
+#risingwave_meta::barrier::progress=debug
+#risingwave_meta::controller::streaming_job=trace
 
 # ========= Some tips for debugging recovery tests =========
 # 1. If materialized view failed to create after multiple retries
@@ -44,11 +45,26 @@ filter_stack_trace_for_all_logs() {
 
 trap filter_stack_trace_for_all_logs ERR
 
+# NOTE(kwannoel): We must use `export` here, because the variables are not substituted
+# directly via bash subtitution. Instead, the `parallel` command substitutes the variables
+# from the environment. If they are declared without `export`, `parallel` can't read them from the env.
+export EXTRA_ARGS=""
+
+if [[ -n "${USE_SQL_BACKEND:-}" ]]; then
+  export EXTRA_ARGS="--sqlite-data-dir=."
+fi
+
+if [[ -n "${USE_ARRANGEMENT_BACKFILL:-}" ]]; then
+  export EXTRA_ARGS="$EXTRA_ARGS --use-arrangement-backfill"
+fi
+
+echo "--- EXTRA_ARGS: ${EXTRA_ARGS}"
+
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, background_ddl"
 seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
-${USE_ARRANGEMENT_BACKFILL:-} \
+${EXTRA_ARGS:-} \
 ./e2e_test/background_ddl/sim/basic.slt \
 2> $LOGDIR/recovery-background-ddl-{}.log && rm $LOGDIR/recovery-background-ddl-{}.log'
 
@@ -57,29 +73,29 @@ seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --background-ddl-rate=${BACKGROUND_DDL_RATE} \
-${USE_ARRANGEMENT_BACKFILL:-} \
+${EXTRA_ARGS:-} \
 ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, streaming"
 seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --background-ddl-rate=${BACKGROUND_DDL_RATE} \
-${USE_ARRANGEMENT_BACKFILL:-} \
+${EXTRA_ARGS:-} \
 ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, batch"
 seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --background-ddl-rate=${BACKGROUND_DDL_RATE} \
-${USE_ARRANGEMENT_BACKFILL:-} \
+${EXTRA_ARGS:-} \
 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, kafka source,sink"
 seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --kafka-datadir=./scripts/source/test_data \
-${USE_ARRANGEMENT_BACKFILL:-} \
+${EXTRA_ARGS:-} \
 ./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log'