From d5e832cbe0339f5deedab3baa18b8f0df8f91772 Mon Sep 17 00:00:00 2001 From: Kevin Axel Date: Mon, 19 Aug 2024 20:38:29 +0800 Subject: [PATCH] feat(test): switch meta to sql backend in simluate test (#15612) Signed-off-by: Kevin Axel Co-authored-by: Noel Kwan Co-authored-by: Noel Kwan <47273164+kwannoel@users.noreply.github.com> --- .gitignore | 3 + Cargo.lock | 108 ++++++++++++---------- Cargo.toml | 6 +- ci/scripts/deterministic-recovery-test.sh | 28 ++++-- ci/workflows/main-cron.yml | 26 +++++- src/object_store/Cargo.toml | 2 +- src/tests/simulation/Cargo.toml | 3 +- src/tests/simulation/src/cluster.rs | 85 ++++++++++++----- src/tests/simulation/src/main.rs | 7 +- 9 files changed, 182 insertions(+), 86 deletions(-) diff --git a/.gitignore b/.gitignore index 25397584e2ef5..72e3ed1487fe0 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,9 @@ e2e_test/iceberg/spark-*-bin* **/poetry.lock +**/*.sqlite +**/*.sqlite-journal + *.slt.temp .direnv/ diff --git a/Cargo.lock b/Cargo.lock index 0bcef68bbe6a4..d007b6c6f4f3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1227,6 +1227,16 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1181e1e0d1fce796a03db1ae795d67167da795f9cf4a39c37589e85ef57f26d3" +[[package]] +name = "atomic-write-file" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf54d4588732bdfc5ebc3eb9f74f20e027112fc31de412fc7ff0cd1c6896dae" +dependencies = [ + "nix 0.28.0", + "rand", +] + [[package]] name = "auto_enums" version = "0.8.5" @@ -2159,7 +2169,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" dependencies = [ "borsh-derive", - "cfg_aliases", + "cfg_aliases 0.2.1", ] [[package]] @@ -2516,6 +2526,12 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "cfg_aliases" version = "0.2.1" @@ -4741,18 +4757,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "flume" -version = "0.10.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" -dependencies = [ - "futures-core", - "futures-sink", - "pin-project", - "spin 0.9.8", -] - [[package]] name = "flume" version = "0.11.0" @@ -6611,9 +6615,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" dependencies = [ "cc", "pkg-config", @@ -6905,9 +6909,9 @@ dependencies = [ [[package]] name = "madsim-tokio" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "569929c869275edc1e2c1f1381a688a6e5a4200302b58caff819e07414ccddb9" +checksum = "53a36782df7996da4863e04526454fbd43067fac1a4dcb8fc663acd538cbf6af" dependencies = [ "madsim", "spin 0.9.8", @@ -7453,6 +7457,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "cfg_aliases 0.1.1", + "libc", +] + [[package]] name = "nix" version = "0.29.0" @@ -7461,7 +7477,7 @@ checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.6.0", "cfg-if", - "cfg_aliases", + "cfg_aliases 0.2.1", "libc", ] @@ -9040,7 +9056,7 @@ checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.11.0", + "itertools 0.10.5", "log", "multimap 0.8.3", "once_cell", @@ -9095,7 +9111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.66", @@ -9278,7 +9294,7 @@ dependencies = [ "indoc", "libc", "memoffset", - "parking_lot 0.12.1", + "parking_lot 0.11.2", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -11496,6 +11512,7 @@ dependencies = [ "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)", "tracing", "tracing-subscriber", + "uuid", ] [[package]] @@ -11843,7 +11860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1568e15fab2d546f940ed3a21f48bbbd1c494c90c99c4481339364a497f94a9" dependencies = [ "bytes", - "flume 0.11.0", + "flume", "futures-util", "log", "rustls-native-certs 0.7.0", @@ -13243,9 +13260,8 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e58421b6bc416714d5115a2ca953718f6c621a51b68e4f4922aea5a4391a721" +version = "0.7.3" +source = "git+https://github.com/kwannoel/sqlx.git?rev=5a32ad8ce842176c202df30910e1416655eda071#5a32ad8ce842176c202df30910e1416655eda071" dependencies = [ "sqlx-core", "sqlx-macros", @@ -13256,9 +13272,8 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4cef4251aabbae751a3710927945901ee1d97ee96d757f6880ebb9a79bfd53" +version = "0.7.3" +source = "git+https://github.com/kwannoel/sqlx.git?rev=5a32ad8ce842176c202df30910e1416655eda071#5a32ad8ce842176c202df30910e1416655eda071" dependencies = [ "ahash 0.8.11", "atoi", @@ -13268,7 +13283,6 @@ dependencies = [ "chrono", "crc", "crossbeam-queue", - "dotenvy", "either", "event-listener 2.5.3", "futures-channel", @@ -13280,6 +13294,7 @@ dependencies = [ "hex", "indexmap 2.2.6", "log", + "madsim-tokio", "memchr", "native-tls", "once_cell", @@ -13293,7 +13308,6 @@ dependencies = [ "sqlformat", "thiserror", "time", - "tokio", "tokio-stream 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", "tracing", "url", @@ -13302,9 +13316,8 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "208e3165167afd7f3881b16c1ef3f2af69fa75980897aac8874a0696516d12c2" +version = "0.7.3" +source = "git+https://github.com/kwannoel/sqlx.git?rev=5a32ad8ce842176c202df30910e1416655eda071#5a32ad8ce842176c202df30910e1416655eda071" dependencies = [ "proc-macro2", "quote", @@ -13315,14 +13328,15 @@ dependencies = [ [[package]] name = "sqlx-macros-core" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a4a8336d278c62231d87f24e8a7a74898156e34c1c18942857be2acb29c7dfc" +version = "0.7.3" +source = "git+https://github.com/kwannoel/sqlx.git?rev=5a32ad8ce842176c202df30910e1416655eda071#5a32ad8ce842176c202df30910e1416655eda071" dependencies = [ + "atomic-write-file", "dotenvy", "either", "heck 0.4.1", "hex", + "madsim-tokio", "once_cell", "proc-macro2", "quote", @@ -13335,15 +13349,13 @@ dependencies = [ "sqlx-sqlite", "syn 1.0.109", "tempfile", - "tokio", "url", ] [[package]] name = "sqlx-mysql" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" +version = "0.7.3" +source = "git+https://github.com/kwannoel/sqlx.git?rev=5a32ad8ce842176c202df30910e1416655eda071#5a32ad8ce842176c202df30910e1416655eda071" dependencies = [ "atoi", "base64 0.21.7", @@ -13388,9 +13400,8 @@ dependencies = [ [[package]] name = "sqlx-postgres" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" +version = "0.7.3" +source = "git+https://github.com/kwannoel/sqlx.git?rev=5a32ad8ce842176c202df30910e1416655eda071#5a32ad8ce842176c202df30910e1416655eda071" dependencies = [ "atoi", "base64 0.21.7", @@ -13433,26 +13444,27 @@ dependencies = [ [[package]] name = "sqlx-sqlite" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4c21bf34c7cae5b283efb3ac1bcc7670df7561124dc2f8bdc0b59be40f79a2" +version = "0.7.3" +source = "git+https://github.com/kwannoel/sqlx.git?rev=5a32ad8ce842176c202df30910e1416655eda071#5a32ad8ce842176c202df30910e1416655eda071" dependencies = [ + "async-channel 2.2.1", "atoi", "chrono", - "flume 0.10.14", + "flume", "futures-channel", "futures-core", - "futures-executor", "futures-intrusive", "futures-util", "libsqlite3-sys", "log", + "madsim-tokio", "percent-encoding", "serde", "sqlx-core", "time", "tracing", "url", + "urlencoding", "uuid", ] diff --git a/Cargo.toml b/Cargo.toml index 35927b31aa783..8c468bf059d50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -193,7 +193,7 @@ sea-orm = { version = "0.12.14", features = [ "sqlx-sqlite", "runtime-tokio-native-tls", ] } -sqlx = "0.7" +sqlx = { version = "0.7.3", default-features = false, features = ["bigdecimal", "chrono", "json", "mysql", "postgres", "runtime-tokio-native-tls", "rust_decimal", "sqlite", "time", "uuid"] } tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "0dd1055", features = ["net", "fs"] } tokio-util = "0.7" tracing-opentelemetry = "0.24" @@ -340,6 +340,10 @@ getrandom = { git = "https://github.com/madsim-rs/getrandom.git", rev = "e79a7ae # tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "0dd1055" } tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = "95e2fd3" } tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88" } +# NOTE(kwannoel): patch sqlx to use a version compatible with madsim. +# If we can merge in: https://github.com/madsim-rs/sqlx/pull/2, +# we can change it to patch madsim version instead. +sqlx = { git = "https://github.com/kwannoel/sqlx.git", rev = "5a32ad8ce842176c202df30910e1416655eda071" } futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" } # patch to remove preserve_order from serde_json bson = { git = "https://github.com/risingwavelabs/bson-rust", rev = "e5175ec" } diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 2afe2f03b956b..ced644f4e4879 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -19,7 +19,8 @@ risingwave_meta::barrier::progress=debug" # Extra logs you can enable if the existing trace does not give enough info. #risingwave_stream::executor::backfill=trace, -#risingwave_meta::barrier::progress=debug, +#risingwave_meta::barrier::progress=debug +#risingwave_meta::controller::streaming_job=trace # ========= Some tips for debugging recovery tests ========= # 1. If materialized view failed to create after multiple retries @@ -44,11 +45,26 @@ filter_stack_trace_for_all_logs() { trap filter_stack_trace_for_all_logs ERR +# NOTE(kwannoel): We must use `export` here, because the variables are not substituted +# directly via bash subtitution. Instead, the `parallel` command substitutes the variables +# from the environment. If they are declared without `export`, `parallel` can't read them from the env. +export EXTRA_ARGS="" + +if [[ -n "${USE_SQL_BACKEND:-}" ]]; then + export EXTRA_ARGS="--sqlite-data-dir=." +fi + +if [[ -n "${USE_ARRANGEMENT_BACKFILL:-}" ]]; then + export EXTRA_ARGS="$EXTRA_ARGS --use-arrangement-backfill" +fi + +echo "--- EXTRA_ARGS: ${EXTRA_ARGS}" + echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, background_ddl" seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ -${USE_ARRANGEMENT_BACKFILL:-} \ +${EXTRA_ARGS:-} \ ./e2e_test/background_ddl/sim/basic.slt \ 2> $LOGDIR/recovery-background-ddl-{}.log && rm $LOGDIR/recovery-background-ddl-{}.log' @@ -57,7 +73,7 @@ seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ --background-ddl-rate=${BACKGROUND_DDL_RATE} \ -${USE_ARRANGEMENT_BACKFILL:-} \ +${EXTRA_ARGS:-} \ ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, streaming" @@ -65,7 +81,7 @@ seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ --background-ddl-rate=${BACKGROUND_DDL_RATE} \ -${USE_ARRANGEMENT_BACKFILL:-} \ +${EXTRA_ARGS:-} \ ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, batch" @@ -73,7 +89,7 @@ seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ --background-ddl-rate=${BACKGROUND_DDL_RATE} \ -${USE_ARRANGEMENT_BACKFILL:-} \ +${EXTRA_ARGS:-} \ ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, kafka source,sink" @@ -81,5 +97,5 @@ seq "$TEST_NUM" | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ --kafka-datadir=./scripts/source/test_data \ -${USE_ARRANGEMENT_BACKFILL:-} \ +${EXTRA_ARGS:-} \ ./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log' \ No newline at end of file diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index 34155618bf9ac..ac5b27eee773d 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -367,7 +367,27 @@ steps: timeout_in_minutes: 80 retry: *auto-retry - - label: "recovery test (madsim)" + # sql backend recovery tests + - label: "recovery test (sql,madsim)" + key: "recovery-test-deterministic-sql" + command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 USE_SQL_BACKEND=true timeout 65m ci/scripts/deterministic-recovery-test.sh" + # NOTE(kwannoel): It will only run when the recovery tests label is added currently. + # This is because there are currently some bugs which cause the test to fail. + if: | + build.pull_request.labels includes "ci/run-sql-recovery-test-deterministic-simulation" + || build.env("CI_STEPS") =~ /(^|,)sql-recovery-tests?-deterministic-simulation(,|$$)/ + depends_on: "build-simulation" + plugins: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + # Only upload zipped files, otherwise the logs is too much. + - ./ci/plugins/upload-failure-logs-zipped + timeout_in_minutes: 70 + retry: *auto-retry + + - label: "recovery test (etcd,madsim)" key: "recovery-test-deterministic" command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 timeout 65m ci/scripts/deterministic-recovery-test.sh" if: | @@ -386,9 +406,9 @@ steps: retry: *auto-retry # Ddl statements will randomly run with background_ddl. - - label: "background_ddl, arrangement_backfill recovery test (madsim)" + - label: "background_ddl, arrangement_backfill recovery test (etcd,madsim)" key: "background-ddl-arrangement-backfill-recovery-test-deterministic" - command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 USE_ARRANGEMENT_BACKFILL=--use-arrangement-backfill timeout 65m ci/scripts/deterministic-recovery-test.sh" + command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 USE_ARRANGEMENT_BACKFILL=true timeout 65m ci/scripts/deterministic-recovery-test.sh" if: | !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation" diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml index fa433a30abcb4..912eb4073130e 100644 --- a/src/object_store/Cargo.toml +++ b/src/object_store/Cargo.toml @@ -30,7 +30,7 @@ hyper = { version = "0.14", features = ["tcp", "client"] } # hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] } hyper-tls = "0.5.0" itertools = { workspace = true } -madsim = "0.2.27" +madsim = "0.2.30" opendal = { workspace = true, features = [ "executors-tokio", "services-azblob", diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index 143d0f0c01c75..8729207c0d025 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -24,7 +24,7 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] } glob = "0.3" itertools = { workspace = true } lru = { workspace = true } -madsim = "0.2.27" +madsim = "0.2.30" paste = "1" pin-project = "1.1" pretty_assertions = "1" @@ -58,6 +58,7 @@ tokio-postgres = "0.7" tokio-stream = { workspace = true } tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +uuid = "*" [lints] workspace = true diff --git a/src/tests/simulation/src/cluster.rs b/src/tests/simulation/src/cluster.rs index 6c9db8c48170f..26fdc3a8757e1 100644 --- a/src/tests/simulation/src/cluster.rs +++ b/src/tests/simulation/src/cluster.rs @@ -39,6 +39,7 @@ use risingwave_pb::common::WorkerNode; use sqllogictest::AsyncDB; #[cfg(not(madsim))] use tokio::runtime::Handle; +use uuid::Uuid; use crate::client::RisingWave; @@ -93,6 +94,9 @@ pub struct Configuration { /// Queries to run per session. pub per_session_queries: Arc>, + + /// dir to store SQL backend sqlite db + pub sqlite_data_dir: Option, } impl Default for Configuration { @@ -122,6 +126,7 @@ metrics_level = "Disabled" etcd_timeout_rate: 0.0, etcd_data_path: None, per_session_queries: vec![].into(), + sqlite_data_dir: None, } } } @@ -356,10 +361,16 @@ impl Cluster { println!("seed = {}", handle.seed()); println!("{:#?}", conf); + if conf.sqlite_data_dir.is_some() && conf.etcd_data_path.is_some() { + bail!("sqlite_data_dir and etcd_data_path cannot be set at the same time"); + } + // setup DNS and load balance let net = madsim::net::NetSim::current(); - net.add_dns_record("etcd", "192.168.10.1".parse().unwrap()); for i in 1..=conf.meta_nodes { + if conf.sqlite_data_dir.is_none() { + net.add_dns_record("etcd", "192.168.10.1".parse().unwrap()); + } net.add_dns_record( &format!("meta-{i}"), format!("192.168.1.{i}").parse().unwrap(), @@ -380,24 +391,26 @@ impl Cluster { } // etcd node - let etcd_data = conf - .etcd_data_path - .as_ref() - .map(|path| std::fs::read_to_string(path).unwrap()); - handle - .create_node() - .name("etcd") - .ip("192.168.10.1".parse().unwrap()) - .init(move || { - let addr = "0.0.0.0:2388".parse().unwrap(); - let mut builder = - etcd_client::SimServer::builder().timeout_rate(conf.etcd_timeout_rate); - if let Some(data) = &etcd_data { - builder = builder.load(data.clone()); - } - builder.serve(addr) - }) - .build(); + if conf.sqlite_data_dir.is_none() { + let etcd_data = conf + .etcd_data_path + .as_ref() + .map(|path| std::fs::read_to_string(path).unwrap()); + handle + .create_node() + .name("etcd") + .ip("192.168.10.1".parse().unwrap()) + .init(move || { + let addr = "0.0.0.0:2388".parse().unwrap(); + let mut builder = + etcd_client::SimServer::builder().timeout_rate(conf.etcd_timeout_rate); + if let Some(data) = &etcd_data { + builder = builder.load(data.clone()); + } + builder.serve(addr) + }) + .build(); + } // kafka broker handle @@ -432,9 +445,33 @@ impl Cluster { } std::env::set_var("RW_META_ADDR", meta_addrs.join(",")); + let mut sql_endpoint = String::new(); + let mut backend_args = if let Some(sqlite_data_dir) = conf.sqlite_data_dir.as_ref() { + sql_endpoint = format!( + "sqlite://{}stest-{}.sqlite?mode=rwc", + sqlite_data_dir.display(), + Uuid::new_v4() + ); + vec!["--backend", "sql", "--sql-endpoint", &sql_endpoint] + } else { + vec!["--backend", "etcd", "--etcd-endpoints", "etcd:2388"] + }; + + // FIXME(kwannoel): + // Currently we just use the on-disk version, + // but it can lead to randomness due to disk io. + // We can use shared in-memory db instead. + // However sqlite cannot be started inside meta. + // Because if cluster stops, then this db will be dropped. + // We must instantiate it outside, not just pass the path in. + // let sqlite_path = format!( + // "sqlite::file:memdb{}?mode=memory&cache=shared", + // Uuid::new_v4() + // ); + // meta node for i in 1..=conf.meta_nodes { - let opts = risingwave_meta_node::MetaNodeOpts::parse_from([ + let args = [ "meta-node", "--config-path", conf.config_path.as_str(), @@ -442,17 +479,15 @@ impl Cluster { "0.0.0.0:5690", "--advertise-addr", &format!("meta-{i}:5690"), - "--backend", - "etcd", - "--etcd-endpoints", - "etcd:2388", "--state-store", "hummock+sim://hummockadmin:hummockadmin@192.168.12.1:9301/hummock001", "--data-directory", "hummock_001", "--temp-secret-file-dir", &format!("./secrets/meta-{i}"), - ]); + ]; + let args = args.into_iter().chain(backend_args.clone().into_iter()); + let opts = risingwave_meta_node::MetaNodeOpts::parse_from(args); handle .create_node() .name(format!("meta-{i}")) diff --git a/src/tests/simulation/src/main.rs b/src/tests/simulation/src/main.rs index e45c4a3285ac3..102db8ccc1c41 100644 --- a/src/tests/simulation/src/main.rs +++ b/src/tests/simulation/src/main.rs @@ -136,6 +136,10 @@ pub struct Args { #[clap(long)] etcd_dump: Option, + /// dir to store sqlite backend data of meta node + #[clap(long)] + sqlite_data_dir: Option, + #[arg(short, long)] e2e_extended_test: bool, @@ -144,7 +148,7 @@ pub struct Args { #[clap(long, default_value = "0.0")] background_ddl_rate: f64, - /// Use arrangement backfill by default + /// Use arrangement backfill #[clap(long, default_value = "false")] use_arrangement_backfill: bool, } @@ -176,6 +180,7 @@ async fn main() { meta_nodes: args.meta_nodes, etcd_timeout_rate: args.etcd_timeout_rate, etcd_data_path: args.etcd_data, + sqlite_data_dir: args.sqlite_data_dir, per_session_queries: if args.use_arrangement_backfill { vec!["SET STREAMING_USE_ARRANGEMENT_BACKFILL = true;".to_string()].into() } else {