diff --git a/Makefile.toml b/Makefile.toml index 37d11266f6d60..604b7b2b4e44b 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -558,8 +558,8 @@ env_files = ["${PREFIX_CONFIG}/risedev-env"] script = ''' #!/usr/bin/env bash cat < "${PREFIX_CONFIG}/psql-env" -export PGHOST=$RW_FRONTEND_LISTEN_ADDRESS -export PGPORT=$RW_FRONTEND_PORT +export PGHOST=$RISEDEV_RW_FRONTEND_LISTEN_ADDRESS +export PGPORT=$RISEDEV_RW_FRONTEND_PORT export PGUSER=root export PGDATABASE=dev EOF @@ -576,7 +576,7 @@ dependencies = ["check-risedev-env-file"] env_files = ["${PREFIX_CONFIG}/risedev-env"] script = ''' #!/usr/bin/env bash -psql -h $RW_FRONTEND_LISTEN_ADDRESS -p $RW_FRONTEND_PORT -U root -d dev "$@" +psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev "$@" ''' [tasks.ctl] @@ -883,7 +883,7 @@ cargo build \ """ [tasks.stest] -category = "RiseDev - Deterministic Simulation" +category = "RiseDev - Test - Deterministic Simulation" description = "Run unit tests in deterministic simulation mode" dependencies = ["install-nextest"] env = { CARGO_TARGET_DIR = "target/sim" } @@ -908,7 +908,7 @@ cargo nextest run \ """ [tasks.sit-test] -category = "RiseDev - Deterministic Simulation" +category = "RiseDev - Test - Deterministic Simulation" description = "Run integration tests in deterministic simulation mode" dependencies = ["install-nextest"] env = { CARGO_TARGET_DIR = "target/sim" } @@ -923,7 +923,7 @@ cargo nextest run \ """ [tasks.sarchive-it-test] -category = "RiseDev - Deterministic Simulation" +category = "RiseDev - Test - Deterministic Simulation" description = "Archive integration tests in deterministic simulation mode" dependencies = ["install-nextest"] env = { CARGO_TARGET_DIR = "target/sim" } @@ -939,7 +939,7 @@ cargo nextest archive \ """ [tasks.scheck] -category = "RiseDev - Deterministic Simulation" +category = "RiseDev - Test - Deterministic Simulation" description = "Run cargo check in deterministic simulation mode" dependencies = ["warn-on-missing-tools"] env = { CARGO_TARGET_DIR = "target/sim" } @@ -964,7 +964,7 @@ cargo check \ """ [tasks.sslt] -category = "RiseDev - Deterministic Simulation" +category = "RiseDev - Test - Deterministic Simulation" description = "Run e2e tests in deterministic simulation mode" dependencies = ["warn-on-missing-tools"] env = { CARGO_TARGET_DIR = "target/sim" } @@ -978,7 +978,7 @@ cargo run \ """ [tasks.sslt-build-all] -category = "RiseDev - Deterministic Simulation" +category = "RiseDev - Test - Deterministic Simulation" description = "Build deterministic simulation runner and tests" dependencies = ["warn-on-missing-tools"] env = { CARGO_TARGET_DIR = "target/sim" } @@ -993,7 +993,7 @@ cargo build \ """ [tasks.sslt-cov] -category = "RiseDev - Deterministic Simulation" +category = "RiseDev - Test - Deterministic Simulation" description = "Run e2e tests in deterministic simulation mode and report code coverage" dependencies = ["install-llvm-cov"] env = { CARGO_TARGET_DIR = "target/sim-cov" } @@ -1290,42 +1290,42 @@ echo "All processes has exited." """ [tasks.slt] -category = "RiseDev - SQLLogicTest" -install_crate = { version = "0.19.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [ +env = { SLT_HOST = "${RISEDEV_RW_FRONTEND_LISTEN_ADDRESS}", SLT_PORT = "${RISEDEV_RW_FRONTEND_PORT}", SLT_DB = "dev" } +category = "RiseDev - Test - SQLLogicTest" +install_crate = { version = "0.20.0", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [ "--help", ], install_command = "binstall" } dependencies = ["check-risedev-env-file"] env_files = ["${PREFIX_CONFIG}/risedev-env"] -env = { SLT_HOST = "${RW_FRONTEND_LISTEN_ADDRESS}", SLT_PORT = "${RW_FRONTEND_PORT}", SLT_DB = "dev" } command = "sqllogictest" args = ["${@}"] description = "🌟 Run SQLLogicTest" [tasks.slt-streaming] -category = "RiseDev - SQLLogicTest" +category = "RiseDev - Test - SQLLogicTest" extend = "slt" args = ["${@}", "./e2e_test/streaming/**/*.slt"] description = "Run all streaming e2e tests" [tasks.slt-batch] -category = "RiseDev - SQLLogicTest" +category = "RiseDev - Test - SQLLogicTest" extend = "slt" args = ["${@}", "./e2e_test/batch/*.slt"] description = "Run all batch e2e tests" [tasks.slt-generated] -category = "RiseDev - SQLLogicTest" +category = "RiseDev - Test - SQLLogicTest" extend = "slt" args = ["${@}", "./e2e_test/generated/**/*.slt"] description = "Run all generated e2e tests" [tasks.slt-all] -category = "RiseDev - SQLLogicTest" +category = "RiseDev - Test - SQLLogicTest" run_task = { name = ["slt-streaming", "slt-batch", "slt-generated"] } description = "Run all e2e tests" [tasks.docslt] -category = "RiseDev - SQLLogicTest" +category = "RiseDev - Test - SQLLogicTest" description = "Extract SQL examples written in SQLLogicTest syntax from Rust doc comments" script = ''' #!/usr/bin/env bash @@ -1400,7 +1400,7 @@ UPDATE_EXPECT=1 cargo test -p risingwave_connector tests::test_with_options_yaml ''' [tasks.backwards-compat-test] -category = "RiseDev - Backwards Compatibility Test" +category = "RiseDev - Test - Backwards Compatibility Test" description = "Run backwards compatibility test" script = "./backwards-compat-tests/scripts/run_local.sh" diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index f3c99c404f6e5..97e1040968604 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -5,15 +5,6 @@ set -euo pipefail source ci/scripts/common.sh -# Arguments: -# $1: subject name -# $2: schema file path -function register_schema_registry() { - curl -X POST http://message_queue:8081/subjects/"$1"/versions \ - -H ‘Content-Type: application/vnd.schemaregistry.v1+json’ \ - --data-binary @<(jq -n --arg schema “$(cat "$2")” ‘{schemaType: “PROTOBUF”, schema: "$schema"}’) -} - # prepare environment export CONNECTOR_LIBS_PATH="./connector-node/libs" @@ -73,26 +64,26 @@ echo 'db.runCommand({ping: 1})' | mongo mongodb://mongodb:27017 echo '> rs config' echo 'rs.conf()' | mongo mongodb://mongodb:27017 echo '> run test..' -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/mongodb/**/*.slt' +risedev slt './e2e_test/source/cdc/mongodb/**/*.slt' echo "--- inline cdc test" export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456 -sqllogictest -p 4566 -d dev './e2e_test/source/cdc_inline/**/*.slt' +risedev slt './e2e_test/source/cdc_inline/**/*.slt' echo "--- opendal source test" -sqllogictest -p 4566 -d dev './e2e_test/source/opendal/**/*.slt' +risedev slt './e2e_test/source/opendal/**/*.slt' echo "--- mysql & postgres cdc validate test" -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.mysql.slt' -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.postgres.slt' +risedev slt './e2e_test/source/cdc/cdc.validate.mysql.slt' +risedev slt './e2e_test/source/cdc/cdc.validate.postgres.slt' echo "--- cdc share source test" # cdc share stream test cases export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456 -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.share_stream.slt' +risedev slt './e2e_test/source/cdc/cdc.share_stream.slt' # create a share source and check whether heartbeat message is received -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.create_source_job.slt' +risedev slt './e2e_test/source/cdc/cdc.create_source_job.slt' table_id=$(psql -U root -h localhost -p 4566 -d dev -t -c "select id from rw_internal_tables where name like '%mysql_source%';" | xargs); table_count=$(psql -U root -h localhost -p 4566 -d dev -t -c "select count(*) from rw_table(${table_id}, public);" | xargs); if [ "$table_count" -eq 0 ]; then @@ -101,10 +92,10 @@ if [ "$table_count" -eq 0 ]; then fi echo "--- mysql & postgres load and check" -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.load.slt' +risedev slt './e2e_test/source/cdc/cdc.load.slt' # wait for cdc loading sleep 10 -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt' +risedev slt './e2e_test/source/cdc/cdc.check.slt' # kill cluster risedev kill @@ -135,10 +126,10 @@ echo "> wait for cluster recovery finish" sleep 20 echo "> check mviews after cluster recovery" # check results -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check_new_rows.slt' +risedev slt './e2e_test/source/cdc/cdc.check_new_rows.slt' # drop relations -sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc_share_stream_drop.slt' +risedev slt './e2e_test/source/cdc/cdc_share_stream_drop.slt' echo "--- Kill cluster" risedev ci-kill @@ -153,8 +144,8 @@ echo "make sure google/protobuf/source_context.proto is NOT in schema registry" curl --silent 'http://message_queue:8081/subjects'; echo # curl --silent --head -X GET 'http://message_queue:8081/subjects/google%2Fprotobuf%2Fsource_context.proto/versions' | grep 404 curl --silent 'http://message_queue:8081/subjects' | grep -v 'google/protobuf/source_context.proto' -sqllogictest -p 4566 -d dev './e2e_test/schema_registry/pb.slt' -sqllogictest -p 4566 -d dev './e2e_test/schema_registry/alter_sr.slt' +risedev slt './e2e_test/schema_registry/pb.slt' +risedev slt './e2e_test/schema_registry/alter_sr.slt' echo "--- Kill cluster" risedev ci-kill @@ -164,19 +155,19 @@ RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=i risedev ci-start ci-pubsub ./scripts/source/prepare_ci_kafka.sh cargo run --bin prepare_ci_pubsub -sqllogictest -p 4566 -d dev './e2e_test/source/basic/*.slt' -sqllogictest -p 4566 -d dev './e2e_test/source/basic/old_row_format_syntax/*.slt' -sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka.slt' +risedev slt './e2e_test/source/basic/*.slt' +risedev slt './e2e_test/source/basic/old_row_format_syntax/*.slt' +risedev slt './e2e_test/source/basic/alter/kafka.slt' echo "--- e2e, kafka alter source" chmod +x ./scripts/source/prepare_data_after_alter.sh ./scripts/source/prepare_data_after_alter.sh 2 -sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka_after_new_data.slt' +risedev slt './e2e_test/source/basic/alter/kafka_after_new_data.slt' echo "--- e2e, kafka alter source again" ./scripts/source/prepare_data_after_alter.sh 3 -sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka_after_new_data_2.slt' +risedev slt './e2e_test/source/basic/alter/kafka_after_new_data_2.slt' echo "--- Run CH-benCHmark" -risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/batch/ch_benchmark.slt' -risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/streaming/*.slt' +risedev slt './e2e_test/ch_benchmark/batch/ch_benchmark.slt' +risedev slt './e2e_test/ch_benchmark/streaming/*.slt' diff --git a/src/cmd_all/scripts/e2e-full-standalone-demo.sh b/src/cmd_all/scripts/e2e-full-standalone-demo.sh index 6c8c01740731b..28469aaddbe70 100755 --- a/src/cmd_all/scripts/e2e-full-standalone-demo.sh +++ b/src/cmd_all/scripts/e2e-full-standalone-demo.sh @@ -67,8 +67,8 @@ sleep 15 # FIXME: Integrate standalone into risedev, so we can reuse risedev-env functionality here. cat << EOF > "$RW_PREFIX"/config/risedev-env RW_META_ADDR="http://0.0.0.0:5690" -RW_FRONTEND_LISTEN_ADDRESS="0.0.0.0" -RW_FRONTEND_PORT="4566" +RISEDEV_RW_FRONTEND_LISTEN_ADDRESS="0.0.0.0" +RISEDEV_RW_FRONTEND_PORT="4566" EOF echo "--- Setting up table" diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs index f4866353dff7e..077f1ce51f82f 100644 --- a/src/risedevtool/src/risedev_env.rs +++ b/src/risedevtool/src/risedev_env.rs @@ -25,50 +25,53 @@ use crate::{add_hummock_backend, HummockInMemoryStrategy, ServiceConfig}; pub fn generate_risedev_env(services: &Vec) -> String { let mut env = String::new(); for item in services { - if let ServiceConfig::ComputeNode(c) = item { - // RW_HUMMOCK_URL - // If the cluster is launched without a shared storage, we will skip this. - { - let mut cmd = Command::new("compute-node"); - if add_hummock_backend( - "dummy", - c.provide_opendal.as_ref().unwrap(), - c.provide_minio.as_ref().unwrap(), - c.provide_aws_s3.as_ref().unwrap(), - HummockInMemoryStrategy::Disallowed, - &mut cmd, - ) - .is_ok() + match item { + ServiceConfig::ComputeNode(c) => { + // RW_HUMMOCK_URL + // If the cluster is launched without a shared storage, we will skip this. + { + let mut cmd = Command::new("compute-node"); + if add_hummock_backend( + "dummy", + c.provide_opendal.as_ref().unwrap(), + c.provide_minio.as_ref().unwrap(), + c.provide_aws_s3.as_ref().unwrap(), + HummockInMemoryStrategy::Disallowed, + &mut cmd, + ) + .is_ok() + { + writeln!( + env, + "RW_HUMMOCK_URL=\"{}\"", + cmd.get_args().nth(1).unwrap().to_str().unwrap() + ) + .unwrap(); + } + } + + // RW_META_ADDR { + let meta_node = &c.provide_meta_node.as_ref().unwrap()[0]; writeln!( env, - "RW_HUMMOCK_URL=\"{}\"", - cmd.get_args().nth(1).unwrap().to_str().unwrap() + "RW_META_ADDR=\"http://{}:{}\"", + meta_node.address, meta_node.port ) .unwrap(); } } - - // RW_META_ADDR - { - let meta_node = &c.provide_meta_node.as_ref().unwrap()[0]; + ServiceConfig::Frontend(c) => { + let listen_address = &c.listen_address; writeln!( env, - "RW_META_ADDR=\"http://{}:{}\"", - meta_node.address, meta_node.port + "RISEDEV_RW_FRONTEND_LISTEN_ADDRESS=\"{listen_address}\"", ) .unwrap(); + let port = &c.port; + writeln!(env, "RISEDEV_RW_FRONTEND_PORT=\"{port}\"",).unwrap(); } - break; - } - } - for item in services { - if let ServiceConfig::Frontend(c) = item { - let listen_address = &c.listen_address; - writeln!(env, "RW_FRONTEND_LISTEN_ADDRESS=\"{listen_address}\"",).unwrap(); - let port = &c.port; - writeln!(env, "RW_FRONTEND_PORT=\"{port}\"",).unwrap(); - break; + _ => {} } } env