diff --git a/Cargo.lock b/Cargo.lock index 315c986ed6709..b2875296b683a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3887,7 +3887,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "tokio", "toml 0.7.8", "url", @@ -4040,12 +4040,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "iter-chunks" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7abddfc4e19bc38f3922e41b341fedb4e1470e922f024c4e5ae5922f56c7593" - [[package]] name = "itertools" version = "0.10.5" @@ -4523,7 +4517,7 @@ dependencies = [ "http", "madsim", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "spin 0.9.8", "thiserror", "tokio", @@ -4597,9 +4591,9 @@ dependencies = [ [[package]] name = "madsim-tonic-build" -version = "0.4.1+0.10.0" +version = "0.4.2+0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3434b3d59001abcce56b9170fbd7982377858d8b931e8472056bf0c894ab257" +checksum = "4a2ad2776ba20221ccbe4e136e2fa0f7ab90eebd608373177f3e74a198a288ec" dependencies = [ "prettyplease 0.2.15", "proc-macro2", @@ -6218,7 +6212,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.15", + "rustix 0.36.16", ] [[package]] @@ -6231,7 +6225,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.15", + "rustix 0.36.16", ] [[package]] @@ -6361,7 +6355,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.37", - "workspace-hack", ] [[package]] @@ -6855,7 +6848,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "serde_yaml", "tempfile", "tracing", @@ -6904,7 +6897,6 @@ dependencies = [ "anyhow", "assert_matches", "async-recursion", - "async-stream", "async-trait", "criterion", "either", @@ -6938,7 +6930,6 @@ dependencies = [ "tokio-metrics", "tokio-stream", "tracing", - "uuid", "workspace-hack", ] @@ -6988,7 +6979,7 @@ dependencies = [ "risingwave_ctl", "risingwave_expr_impl", "risingwave_frontend", - "risingwave_meta", + "risingwave_meta_node", "risingwave_rt", "task_stats_alloc", "tikv-jemallocator", @@ -7014,7 +7005,7 @@ dependencies = [ "risingwave_ctl", "risingwave_expr_impl", "risingwave_frontend", - "risingwave_meta", + "risingwave_meta_node", "risingwave_rt", "shell-words", "strum 0.25.0", @@ -7100,7 +7091,7 @@ dependencies = [ "serde_bytes", "serde_default", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "smallbitset", "speedate", "static_assertions", @@ -7109,7 +7100,6 @@ dependencies = [ "sysinfo", "tempfile", "thiserror", - "tikv-jemalloc-ctl", "tinyvec", "toml 0.8.2", "tower-layer", @@ -7123,6 +7113,19 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "risingwave_common_heap_profiling" +version = "1.3.0-alpha" +dependencies = [ + "anyhow", + "chrono", + "madsim-tokio", + "parking_lot 0.12.1", + "risingwave_common", + "tikv-jemalloc-ctl", + "tracing", +] + [[package]] name = "risingwave_common_proc_macro" version = "1.3.0-alpha" @@ -7132,7 +7135,6 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "workspace-hack", ] [[package]] @@ -7171,6 +7173,7 @@ dependencies = [ "risingwave_hummock_sdk", "risingwave_hummock_test", "risingwave_meta", + "risingwave_meta_node", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", @@ -7184,22 +7187,20 @@ dependencies = [ name = "risingwave_compactor" version = "1.3.0-alpha" dependencies = [ - "anyhow", "async-trait", "await-tree", "clap", "madsim-tokio", "madsim-tonic", "parking_lot 0.12.1", - "prometheus", "risingwave_common", + "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", "risingwave_storage", "serde", - "serde_json", "tracing", "workspace-hack", ] @@ -7226,9 +7227,11 @@ dependencies = [ "rand", "risingwave_batch", "risingwave_common", + "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_connector", "risingwave_hummock_sdk", + "risingwave_jni_core", "risingwave_pb", "risingwave_rpc_client", "risingwave_source", @@ -7263,7 +7266,6 @@ dependencies = [ "aws-smithy-http", "aws-types", "base64 0.21.4", - "bincode 1.3.3", "byteorder", "bytes", "chrono", @@ -7287,15 +7289,12 @@ dependencies = [ "jsonschema-transpiler", "madsim-rdkafka", "madsim-tokio", - "madsim-tonic", "maplit", "moka", "mysql_async", "mysql_common", "nexmark", - "nkeys", "num-bigint", - "opendal 0.39.0", "parking_lot 0.12.1", "paste", "prometheus", @@ -7317,7 +7316,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "simd-json", "strum 0.25.0", "strum_macros 0.25.2", @@ -7492,7 +7491,6 @@ dependencies = [ "madsim-tonic", "maplit", "md5", - "more-asserts", "num-integer", "parking_lot 0.12.1", "parse-display", @@ -7671,16 +7669,14 @@ dependencies = [ "num-integer", "num-traits", "parking_lot 0.12.1", - "parse-display", "prometheus", "prometheus-http-query", "prost 0.12.1", "rand", - "regex", "reqwest", "risingwave_backup", "risingwave_common", - "risingwave_common_service", + "risingwave_common_heap_profiling", "risingwave_connector", "risingwave_hummock_sdk", "risingwave_object_store", @@ -7693,9 +7689,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "static_assertions", "sync-point", - "tempfile", "thiserror", "tokio-retry", "tokio-stream", @@ -7707,6 +7701,56 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "risingwave_meta_node" +version = "1.3.0-alpha" +dependencies = [ + "anyhow", + "clap", + "either", + "futures", + "itertools 0.11.0", + "madsim-etcd-client", + "madsim-tokio", + "madsim-tonic", + "model_migration", + "prometheus-http-query", + "regex", + "risingwave_common", + "risingwave_common_heap_profiling", + "risingwave_common_service", + "risingwave_meta", + "risingwave_meta_service", + "risingwave_pb", + "risingwave_rpc_client", + "sea-orm", + "tracing", + "workspace-hack", +] + +[[package]] +name = "risingwave_meta_service" +version = "1.3.0-alpha" +dependencies = [ + "anyhow", + "async-trait", + "either", + "futures", + "itertools 0.11.0", + "madsim-tokio", + "madsim-tonic", + "regex", + "risingwave_common", + "risingwave_connector", + "risingwave_meta", + "risingwave_pb", + "sea-orm", + "sync-point", + "tokio-stream", + "tracing", + "workspace-hack", +] + [[package]] name = "risingwave_object_store" version = "1.3.0-alpha" @@ -7767,7 +7811,7 @@ dependencies = [ "risingwave_frontend", "risingwave_sqlparser", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "serde_yaml", "tempfile", "walkdir", @@ -7822,7 +7866,6 @@ name = "risingwave_rt" version = "1.3.0-alpha" dependencies = [ "await-tree", - "chrono", "console", "console-subscriber", "either", @@ -7834,7 +7877,6 @@ dependencies = [ "opentelemetry-semantic-conventions", "parking_lot 0.12.1", "pprof", - "prometheus", "risingwave_common", "risingwave_variables", "rlimit", @@ -7876,7 +7918,7 @@ dependencies = [ "risingwave_e2e_extended_mode_test", "risingwave_expr_impl", "risingwave_frontend", - "risingwave_meta", + "risingwave_meta_node", "risingwave_pb", "risingwave_rpc_client", "risingwave_sqlparser", @@ -7900,7 +7942,6 @@ dependencies = [ "anyhow", "assert_matches", "criterion", - "easy-ext", "futures", "futures-async-stream", "itertools 0.11.0", @@ -7937,7 +7978,7 @@ dependencies = [ "madsim-tokio", "risingwave_sqlparser", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "serde_yaml", "walkdir", "workspace-hack", @@ -7982,7 +8023,7 @@ dependencies = [ "regex", "risingwave_rt", "serde", - "serde_with 3.3.0", + "serde_with 3.4.0", "tokio-postgres", "tokio-stream", "toml 0.8.2", @@ -7994,7 +8035,6 @@ dependencies = [ name = "risingwave_storage" version = "1.3.0-alpha" dependencies = [ - "anyhow", "arc-swap", "async-trait", "auto_enums", @@ -8067,7 +8107,6 @@ dependencies = [ "await-tree", "bytes", "criterion", - "dyn-clone", "educe", "either", "enum-as-inner", @@ -8076,7 +8115,6 @@ dependencies = [ "futures-async-stream", "governor", "hytra", - "iter-chunks", "itertools 0.11.0", "local_stats_alloc", "lru 0.7.6", @@ -8085,9 +8123,7 @@ dependencies = [ "maplit", "memcomparable", "multimap 0.9.0", - "num-traits", "parking_lot 0.12.1", - "parse-display", "pin-project", "prometheus", "prost 0.12.1", @@ -8106,7 +8142,6 @@ dependencies = [ "serde_json", "serde_yaml", "smallvec", - "spin 0.9.8", "static_assertions", "task_stats_alloc", "thiserror", @@ -8265,9 +8300,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.15" +version = "0.36.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941" +checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" dependencies = [ "bitflags 1.3.2", "errno", @@ -8824,9 +8859,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca3b16a3d82c4088f343b7480a93550b3eabe1a358569c2dfe38bbcead07237" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" dependencies = [ "base64 0.21.4", "chrono", @@ -8835,7 +8870,7 @@ dependencies = [ "indexmap 2.0.0", "serde", "serde_json", - "serde_with_macros 3.3.0", + "serde_with_macros 3.4.0", "time", ] @@ -8853,9 +8888,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e6be15c453eb305019bfa438b1593c731f36a289a7853f7707ee29e870b3b3c" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -10456,9 +10491,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ "getrandom", "rand", @@ -10999,7 +11034,7 @@ dependencies = [ "sea-query-binder", "serde", "serde_json", - "serde_with 3.3.0", + "serde_with 3.4.0", "sha1", "sha2", "signature", diff --git a/Cargo.toml b/Cargo.toml index 81d914b416d17..ef09221b818a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "src/cmd_all", "src/common", "src/common/common_service", + "src/common/heap_profiling", "src/compute", "src/connector", "src/ctl", @@ -18,6 +19,8 @@ members = [ "src/java_binding", "src/jni_core", "src/meta", + "src/meta/node", + "src/meta/service", "src/meta/src/model_v2/migration", "src/object_store", "src/prost", @@ -106,7 +109,7 @@ hashbrown = { version = "0.14.0", features = [ ] } criterion = { version = "0.5", features = ["async_futures"] } tonic = { package = "madsim-tonic", version = "0.4.0" } -tonic-build = { package = "madsim-tonic-build", version = "0.4.0" } +tonic-build = { package = "madsim-tonic-build", version = "0.4.2" } prost = { version = "0.12" } icelake = { git = "https://github.com/icelake-io/icelake", rev = "16dab0e36ab337e58ee8002d828def2d212fa116" } arrow-array = "47" @@ -127,6 +130,7 @@ risingwave_batch = { path = "./src/batch" } risingwave_cmd = { path = "./src/cmd" } risingwave_common = { path = "./src/common" } risingwave_common_service = { path = "./src/common/common_service" } +risingwave_common_heap_profiling = { path = "./src/common/heap_profiling" } risingwave_compactor = { path = "./src/storage/compactor" } risingwave_compute = { path = "./src/compute" } risingwave_ctl = { path = "./src/ctl" } @@ -138,6 +142,8 @@ risingwave_hummock_sdk = { path = "./src/storage/hummock_sdk" } risingwave_hummock_test = { path = "./src/storage/hummock_test" } risingwave_hummock_trace = { path = "./src/storage/hummock_trace" } risingwave_meta = { path = "./src/meta" } +risingwave_meta_service = { path = "./src/meta/service" } +risingwave_meta_node = { path = "./src/meta/node" } risingwave_object_store = { path = "./src/object_store" } risingwave_pb = { path = "./src/prost" } risingwave_rpc_client = { path = "./src/rpc_client" } diff --git a/Makefile.toml b/Makefile.toml index a0c6213a63c82..86e9cea136bb5 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -406,6 +406,7 @@ condition = { env_set = [ "ENABLE_BUILD_DASHBOARD", ], files_modified = { input = [ "./dashboard/**/*.js", + "./dashboard/**/*.ts*", "./dashboard/package.json", "./dashboard/next.config.js", ], output = [ @@ -1316,7 +1317,6 @@ dependencies = ["k", "l", "check-logs", "wait-processes-exit"] description = "Kill cluster, dump logs and check logs" [tasks.wait-processes-exit] -private = true category = "Misc" description = "Wait for RisingWave processes to exit" script = """ diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 0d3a7b3fabed4..6514fe1f7c0c3 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -9,11 +9,19 @@ echo "--- Download artifacts" download-and-decompress-artifact risingwave_simulation . chmod +x ./risingwave_simulation -export RUST_LOG="info,risingwave_meta::barrier::recovery=debug" +export RUST_LOG="info,\ +risingwave_meta::barrier::recovery=debug,\ +risingwave_meta::rpc::ddl_controller=debug,\ +risingwave_meta::barrier::mod=debug,\ +risingwave_simulation=debug" export LOGDIR=.risingwave/log mkdir -p $LOGDIR +# FIXME(kwannoel): Why is this failing? +# echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, background_ddl" +# seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/background_ddl/sim/basic.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' + echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, ddl" seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh index 85aad20749d61..06ef185f46e8b 100755 --- a/ci/scripts/e2e-kafka-sink-test.sh +++ b/ci/scripts/e2e-kafka-sink-test.sh @@ -5,6 +5,7 @@ set -euo pipefail ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --create > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --create > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --create > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --create > /dev/null 2>&1 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/create_sink.slt' @@ -28,6 +29,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema +echo "testing upsert kafka sink with schema" +diff ./e2e_test/sink/kafka/upsert_schema1.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink echo "testing debezium kafka sink" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 10 | sort) > ./e2e_test/sink/kafka/debezium1.tmp.result 2> /dev/null @@ -62,6 +72,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema after update +echo "testing upsert kafka sink with schema after updating data" +diff ./e2e_test/sink/kafka/upsert_schema2.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after update echo "testing debezium kafka sink after updating data" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 11 | sort) > ./e2e_test/sink/kafka/debezium2.tmp.result 2> /dev/null @@ -87,6 +106,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema after delete +echo "testing upsert kafka sink with schema after deleting data" +diff ./e2e_test/sink/kafka/upsert_schema3.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after delete echo "testing debezium kafka sink after deleting data" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 13 | sort) > ./e2e_test/sink/kafka/debezium3.tmp.result 2> /dev/null @@ -103,3 +131,10 @@ sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/drop_sink.slt' ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --delete > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --delete > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --delete > /dev/null 2>&1 + +# test different encoding +echo "testing protobuf" +cp src/connector/src/test_data/proto_recursive/recursive.pb ./proto-recursive +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only-protobuf --create > /dev/null 2>&1 +sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/protobuf.slt' +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only-protobuf --delete > /dev/null 2>&1 diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index fee45e6afc4f8..08e5794f173cd 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -11,7 +11,7 @@ if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhoo fi echo "--- Install java and maven" -yum install -y java-11-openjdk wget python3 cyrus-sasl-devel +yum install -y java-11-openjdk java-11-openjdk-devel wget python3 cyrus-sasl-devel pip3 install toml-cli wget https://ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz export PATH="${REPO_ROOT}/apache-maven-3.9.3/bin:$PATH" @@ -64,6 +64,10 @@ elif [[ -n "${BINARY_NAME+x}" ]]; then aws s3 cp risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary fi +echo "--- Build connector node" +cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust +cd ${REPO_ROOT} && mv ${REPO_ROOT}/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz + if [[ -n "${BUILDKITE_TAG}" ]]; then echo "--- Install gh cli" yum install -y dnf @@ -87,8 +91,6 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz echo "--- Release build and upload risingwave connector node jar asset" - cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust - cd connector-node/assembly/target && mv risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz fi diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh index 86dbcb376c255..d0d5eafb3c917 100755 --- a/ci/scripts/run-backfill-tests.sh +++ b/ci/scripts/run-backfill-tests.sh @@ -1,22 +1,31 @@ #!/usr/bin/env bash # Runs backfill tests. -# NOTE(kwannoel): -# The following scenario is adapted in madsim's integration tests as well. -# But this script reproduces it more reliably (I'm not sure why.) -# Hence keeping it in case we ever need to debug backfill again. # USAGE: # ```sh # cargo make ci-start ci-backfill # ./ci/scripts/run-backfill-tests.sh # ``` +# Example progress: +# dev=> select * from rw_catalog.rw_ddl_progress; +# ddl_id | ddl_statement | progress | initialized_at +#--------+------------------------------------------------+----------+------------------------------- +# 1002 | CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t | 56.12% | 2023-09-27 06:37:06.636+00:00 +#(1 row) set -euo pipefail PARENT_PATH=$(dirname "${BASH_SOURCE[0]}") +TEST_DIR=$PWD/e2e_test +BACKGROUND_DDL_DIR=$TEST_DIR/background_ddl +COMMON_DIR=$BACKGROUND_DDL_DIR/common + +CLUSTER_PROFILE='ci-1cn-1fe-with-recovery' +export RUST_LOG="risingwave_meta=debug" + run_sql_file() { psql -h localhost -p 4566 -d dev -U root -f "$@" } @@ -29,40 +38,309 @@ flush() { run_sql "FLUSH;" } -basic() { - echo "--- e2e, test_backfill_basic" +cancel_stream_jobs() { + ID=$(run_sql "select ddl_id from rw_catalog.rw_ddl_progress;" | tail -3 | head -1 | grep -E -o "[0-9]*") + echo "CANCELLING STREAM_JOB: $ID" + run_sql "CANCEL JOBS $ID;" .risingwave/log/compute-node.log 2>&1 & +} + +# Test snapshot and upstream read. +test_snapshot_and_upstream_read() { + echo "--- e2e, ci-backfill, test_snapshot_and_upstream_read" cargo make ci-start ci-backfill + run_sql_file "$PARENT_PATH"/sql/backfill/create_base_table.sql # Provide snapshot run_sql_file "$PARENT_PATH"/sql/backfill/insert.sql + + # Provide updates ... run_sql_file "$PARENT_PATH"/sql/backfill/insert.sql & + + # ... and concurrently create mv. run_sql_file "$PARENT_PATH"/sql/backfill/create_mv.sql & wait run_sql_file "$PARENT_PATH"/sql/backfill/select.sql () - const [dump, setDump] = useState("") + const [computeNodeId, setComputeNodeId] = useState() + const [dump, setDump] = useState("") useEffect(() => { - if (computeNodes && !computeNodeId && computeNodes.length > 0) { - setComputeNodeId(computeNodes[0].id) + if (computeNodes && !computeNodeId) { + setComputeNodeId(ALL_COMPUTE_NODES) } }, [computeNodes, computeNodeId]) const dumpTree = async () => { - const title = `Await-Tree Dump of Compute Node ${computeNodeId}:` - setDump(undefined) + if (computeNodeId === undefined) { + return + } + + let title + if (computeNodeId === ALL_COMPUTE_NODES) { + title = "Await-Tree Dump of All Compute Nodes:" + } else { + title = `Await-Tree Dump of Compute Node ${computeNodeId}:` + } + setDump("Loading...") let result @@ -92,10 +102,13 @@ export default function AwaitTreeDump() { Compute Nodes