style: minor improvements for source test and risedev (#16289)

Signed-off-by: xxchan <[email protected]>
risingwavelabs · Apr 15, 2024 · a3939b0 · a3939b0
1 parent 00d8958
commit a3939b0
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 82 deletions.
diff --git a/Makefile.toml b/Makefile.toml
@@ -558,8 +558,8 @@ env_files = ["${PREFIX_CONFIG}/risedev-env"]
 script = '''
 #!/usr/bin/env bash
 cat <<EOF > "${PREFIX_CONFIG}/psql-env"
-export PGHOST=$RW_FRONTEND_LISTEN_ADDRESS
-export PGPORT=$RW_FRONTEND_PORT
+export PGHOST=$RISEDEV_RW_FRONTEND_LISTEN_ADDRESS
+export PGPORT=$RISEDEV_RW_FRONTEND_PORT
 export PGUSER=root
 export PGDATABASE=dev
 EOF
@@ -576,7 +576,7 @@ dependencies = ["check-risedev-env-file"]
 env_files = ["${PREFIX_CONFIG}/risedev-env"]
 script = '''
 #!/usr/bin/env bash
-psql -h $RW_FRONTEND_LISTEN_ADDRESS -p $RW_FRONTEND_PORT -U root -d dev "$@"
+psql -h $RISEDEV_RW_FRONTEND_LISTEN_ADDRESS -p $RISEDEV_RW_FRONTEND_PORT -U root -d dev "$@"
 '''
 
 [tasks.ctl]
@@ -883,7 +883,7 @@ cargo build \
 """
 
 [tasks.stest]
-category = "RiseDev - Deterministic Simulation"
+category = "RiseDev - Test - Deterministic Simulation"
 description = "Run unit tests in deterministic simulation mode"
 dependencies = ["install-nextest"]
 env = { CARGO_TARGET_DIR = "target/sim" }
@@ -908,7 +908,7 @@ cargo nextest run \
 """
 
 [tasks.sit-test]
-category = "RiseDev - Deterministic Simulation"
+category = "RiseDev - Test - Deterministic Simulation"
 description = "Run integration tests in deterministic simulation mode"
 dependencies = ["install-nextest"]
 env = { CARGO_TARGET_DIR = "target/sim" }
@@ -923,7 +923,7 @@ cargo nextest run \
 """
 
 [tasks.sarchive-it-test]
-category = "RiseDev - Deterministic Simulation"
+category = "RiseDev - Test - Deterministic Simulation"
 description = "Archive integration tests in deterministic simulation mode"
 dependencies = ["install-nextest"]
 env = { CARGO_TARGET_DIR = "target/sim" }
@@ -939,7 +939,7 @@ cargo nextest archive \
 """
 
 [tasks.scheck]
-category = "RiseDev - Deterministic Simulation"
+category = "RiseDev - Test - Deterministic Simulation"
 description = "Run cargo check in deterministic simulation mode"
 dependencies = ["warn-on-missing-tools"]
 env = { CARGO_TARGET_DIR = "target/sim" }
@@ -964,7 +964,7 @@ cargo check \
 """
 
 [tasks.sslt]
-category = "RiseDev - Deterministic Simulation"
+category = "RiseDev - Test - Deterministic Simulation"
 description = "Run e2e tests in deterministic simulation mode"
 dependencies = ["warn-on-missing-tools"]
 env = { CARGO_TARGET_DIR = "target/sim" }
@@ -978,7 +978,7 @@ cargo run \
 """
 
 [tasks.sslt-build-all]
-category = "RiseDev - Deterministic Simulation"
+category = "RiseDev - Test - Deterministic Simulation"
 description = "Build deterministic simulation runner and tests"
 dependencies = ["warn-on-missing-tools"]
 env = { CARGO_TARGET_DIR = "target/sim" }
@@ -993,7 +993,7 @@ cargo build \
 """
 
 [tasks.sslt-cov]
-category = "RiseDev - Deterministic Simulation"
+category = "RiseDev - Test - Deterministic Simulation"
 description = "Run e2e tests in deterministic simulation mode and report code coverage"
 dependencies = ["install-llvm-cov"]
 env = { CARGO_TARGET_DIR = "target/sim-cov" }
@@ -1290,42 +1290,42 @@ echo "All processes has exited."
 """
 
 [tasks.slt]
-category = "RiseDev - SQLLogicTest"
-install_crate = { version = "0.19.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
+env = { SLT_HOST = "${RISEDEV_RW_FRONTEND_LISTEN_ADDRESS}", SLT_PORT = "${RISEDEV_RW_FRONTEND_PORT}", SLT_DB = "dev" }
+category = "RiseDev - Test - SQLLogicTest"
+install_crate = { version = "0.20.0", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
   "--help",
 ], install_command = "binstall" }
 dependencies = ["check-risedev-env-file"]
 env_files = ["${PREFIX_CONFIG}/risedev-env"]
-env = { SLT_HOST = "${RW_FRONTEND_LISTEN_ADDRESS}", SLT_PORT = "${RW_FRONTEND_PORT}", SLT_DB = "dev" }
 command = "sqllogictest"
 args = ["${@}"]
 description = "🌟 Run SQLLogicTest"
 
 [tasks.slt-streaming]
-category = "RiseDev - SQLLogicTest"
+category = "RiseDev - Test - SQLLogicTest"
 extend = "slt"
 args = ["${@}", "./e2e_test/streaming/**/*.slt"]
 description = "Run all streaming e2e tests"
 
 [tasks.slt-batch]
-category = "RiseDev - SQLLogicTest"
+category = "RiseDev - Test - SQLLogicTest"
 extend = "slt"
 args = ["${@}", "./e2e_test/batch/*.slt"]
 description = "Run all batch e2e tests"
 
 [tasks.slt-generated]
-category = "RiseDev - SQLLogicTest"
+category = "RiseDev - Test - SQLLogicTest"
 extend = "slt"
 args = ["${@}", "./e2e_test/generated/**/*.slt"]
 description = "Run all generated e2e tests"
 
 [tasks.slt-all]
-category = "RiseDev - SQLLogicTest"
+category = "RiseDev - Test - SQLLogicTest"
 run_task = { name = ["slt-streaming", "slt-batch", "slt-generated"] }
 description = "Run all e2e tests"
 
 [tasks.docslt]
-category = "RiseDev - SQLLogicTest"
+category = "RiseDev - Test - SQLLogicTest"
 description = "Extract SQL examples written in SQLLogicTest syntax from Rust doc comments"
 script = '''
 #!/usr/bin/env bash
@@ -1400,7 +1400,7 @@ UPDATE_EXPECT=1 cargo test -p risingwave_connector tests::test_with_options_yaml
 '''
 
 [tasks.backwards-compat-test]
-category = "RiseDev - Backwards Compatibility Test"
+category = "RiseDev - Test - Backwards Compatibility Test"
 description = "Run backwards compatibility test"
 script = "./backwards-compat-tests/scripts/run_local.sh"
 

diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
@@ -5,15 +5,6 @@ set -euo pipefail
 
 source ci/scripts/common.sh
 
-# Arguments:
-#   $1: subject name
-#   $2: schema file path
-function register_schema_registry() {
-    curl -X POST http://message_queue:8081/subjects/"$1"/versions \
-        -H ‘Content-Type: application/vnd.schemaregistry.v1+json’ \
-        --data-binary @<(jq -n --arg schema “$(cat "$2")” ‘{schemaType: “PROTOBUF”, schema: "$schema"}’)
-}
-
 # prepare environment
 export CONNECTOR_LIBS_PATH="./connector-node/libs"
 
@@ -73,26 +64,26 @@ echo 'db.runCommand({ping: 1})' | mongo mongodb://mongodb:27017
 echo '> rs config'
 echo 'rs.conf()' | mongo mongodb://mongodb:27017
 echo '> run test..'
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/mongodb/**/*.slt'
+risedev slt './e2e_test/source/cdc/mongodb/**/*.slt'
 
 echo "--- inline cdc test"
 export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc_inline/**/*.slt'
+risedev slt './e2e_test/source/cdc_inline/**/*.slt'
 
 echo "--- opendal source test"
-sqllogictest -p 4566 -d dev './e2e_test/source/opendal/**/*.slt'
+risedev slt './e2e_test/source/opendal/**/*.slt'
 
 echo "--- mysql & postgres cdc validate test"
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.mysql.slt'
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.postgres.slt'
+risedev slt './e2e_test/source/cdc/cdc.validate.mysql.slt'
+risedev slt './e2e_test/source/cdc/cdc.validate.postgres.slt'
 
 echo "--- cdc share source test"
 # cdc share stream test cases
 export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.share_stream.slt'
+risedev slt './e2e_test/source/cdc/cdc.share_stream.slt'
 
 # create a share source and check whether heartbeat message is received
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.create_source_job.slt'
+risedev slt './e2e_test/source/cdc/cdc.create_source_job.slt'
 table_id=$(psql -U root -h localhost -p 4566 -d dev -t -c "select id from rw_internal_tables where name like '%mysql_source%';" | xargs);
 table_count=$(psql -U root -h localhost -p 4566 -d dev -t -c "select count(*) from rw_table(${table_id}, public);" | xargs);
 if [ "$table_count" -eq 0 ]; then
@@ -101,10 +92,10 @@ if [ "$table_count" -eq 0 ]; then
 fi
 
 echo "--- mysql & postgres load and check"
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.load.slt'
+risedev slt './e2e_test/source/cdc/cdc.load.slt'
 # wait for cdc loading
 sleep 10
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check.slt'
+risedev slt './e2e_test/source/cdc/cdc.check.slt'
 
 # kill cluster
 risedev kill
@@ -135,10 +126,10 @@ echo "> wait for cluster recovery finish"
 sleep 20
 echo "> check mviews after cluster recovery"
 # check results
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.check_new_rows.slt'
+risedev slt './e2e_test/source/cdc/cdc.check_new_rows.slt'
 
 # drop relations
-sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc_share_stream_drop.slt'
+risedev slt './e2e_test/source/cdc/cdc_share_stream_drop.slt'
 
 echo "--- Kill cluster"
 risedev ci-kill
@@ -153,8 +144,8 @@ echo "make sure google/protobuf/source_context.proto is NOT in schema registry"
 curl --silent 'http://message_queue:8081/subjects'; echo
 # curl --silent --head -X GET 'http://message_queue:8081/subjects/google%2Fprotobuf%2Fsource_context.proto/versions' | grep 404
 curl --silent 'http://message_queue:8081/subjects' | grep -v 'google/protobuf/source_context.proto'
-sqllogictest -p 4566 -d dev './e2e_test/schema_registry/pb.slt'
-sqllogictest -p 4566 -d dev './e2e_test/schema_registry/alter_sr.slt'
+risedev slt './e2e_test/schema_registry/pb.slt'
+risedev slt './e2e_test/schema_registry/alter_sr.slt'
 
 echo "--- Kill cluster"
 risedev ci-kill
@@ -164,19 +155,19 @@ RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=i
 risedev ci-start ci-pubsub
 ./scripts/source/prepare_ci_kafka.sh
 cargo run --bin prepare_ci_pubsub
-sqllogictest -p 4566 -d dev './e2e_test/source/basic/*.slt'
-sqllogictest -p 4566 -d dev './e2e_test/source/basic/old_row_format_syntax/*.slt'
-sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka.slt'
+risedev slt './e2e_test/source/basic/*.slt'
+risedev slt './e2e_test/source/basic/old_row_format_syntax/*.slt'
+risedev slt './e2e_test/source/basic/alter/kafka.slt'
 
 echo "--- e2e, kafka alter source"
 chmod +x ./scripts/source/prepare_data_after_alter.sh
 ./scripts/source/prepare_data_after_alter.sh 2
-sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka_after_new_data.slt'
+risedev slt './e2e_test/source/basic/alter/kafka_after_new_data.slt'
 
 echo "--- e2e, kafka alter source again"
 ./scripts/source/prepare_data_after_alter.sh 3
-sqllogictest -p 4566 -d dev './e2e_test/source/basic/alter/kafka_after_new_data_2.slt'
+risedev slt './e2e_test/source/basic/alter/kafka_after_new_data_2.slt'
 
 echo "--- Run CH-benCHmark"
-risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
-risedev slt -p 4566 -d dev './e2e_test/ch_benchmark/streaming/*.slt'
+risedev slt './e2e_test/ch_benchmark/batch/ch_benchmark.slt'
+risedev slt './e2e_test/ch_benchmark/streaming/*.slt'
diff --git a/src/cmd_all/scripts/e2e-full-standalone-demo.sh b/src/cmd_all/scripts/e2e-full-standalone-demo.sh
@@ -67,8 +67,8 @@ sleep 15
 # FIXME: Integrate standalone into risedev, so we can reuse risedev-env functionality here.
 cat << EOF > "$RW_PREFIX"/config/risedev-env
 RW_META_ADDR="http://0.0.0.0:5690"
-RW_FRONTEND_LISTEN_ADDRESS="0.0.0.0"
-RW_FRONTEND_PORT="4566"
+RISEDEV_RW_FRONTEND_LISTEN_ADDRESS="0.0.0.0"
+RISEDEV_RW_FRONTEND_PORT="4566"
 EOF
 
 echo "--- Setting up table"

diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs
@@ -25,50 +25,53 @@ use crate::{add_hummock_backend, HummockInMemoryStrategy, ServiceConfig};
 pub fn generate_risedev_env(services: &Vec<ServiceConfig>) -> String {
     let mut env = String::new();
     for item in services {
-        if let ServiceConfig::ComputeNode(c) = item {
-            // RW_HUMMOCK_URL
-            // If the cluster is launched without a shared storage, we will skip this.
-            {
-                let mut cmd = Command::new("compute-node");
-                if add_hummock_backend(
-                    "dummy",
-                    c.provide_opendal.as_ref().unwrap(),
-                    c.provide_minio.as_ref().unwrap(),
-                    c.provide_aws_s3.as_ref().unwrap(),
-                    HummockInMemoryStrategy::Disallowed,
-                    &mut cmd,
-                )
-                .is_ok()
+        match item {
+            ServiceConfig::ComputeNode(c) => {
+                // RW_HUMMOCK_URL
+                // If the cluster is launched without a shared storage, we will skip this.
+                {
+                    let mut cmd = Command::new("compute-node");
+                    if add_hummock_backend(
+                        "dummy",
+                        c.provide_opendal.as_ref().unwrap(),
+                        c.provide_minio.as_ref().unwrap(),
+                        c.provide_aws_s3.as_ref().unwrap(),
+                        HummockInMemoryStrategy::Disallowed,
+                        &mut cmd,
+                    )
+                    .is_ok()
+                    {
+                        writeln!(
+                            env,
+                            "RW_HUMMOCK_URL=\"{}\"",
+                            cmd.get_args().nth(1).unwrap().to_str().unwrap()
+                        )
+                        .unwrap();
+                    }
+                }
+
+                // RW_META_ADDR
                 {
+                    let meta_node = &c.provide_meta_node.as_ref().unwrap()[0];
                     writeln!(
                         env,
-                        "RW_HUMMOCK_URL=\"{}\"",
-                        cmd.get_args().nth(1).unwrap().to_str().unwrap()
+                        "RW_META_ADDR=\"http://{}:{}\"",
+                        meta_node.address, meta_node.port
                     )
                     .unwrap();
                 }
             }
-
-            // RW_META_ADDR
-            {
-                let meta_node = &c.provide_meta_node.as_ref().unwrap()[0];
+            ServiceConfig::Frontend(c) => {
+                let listen_address = &c.listen_address;
                 writeln!(
                     env,
-                    "RW_META_ADDR=\"http://{}:{}\"",
-                    meta_node.address, meta_node.port
+                    "RISEDEV_RW_FRONTEND_LISTEN_ADDRESS=\"{listen_address}\"",
                 )
                 .unwrap();
+                let port = &c.port;
+                writeln!(env, "RISEDEV_RW_FRONTEND_PORT=\"{port}\"",).unwrap();
             }
-            break;
-        }
-    }
-    for item in services {
-        if let ServiceConfig::Frontend(c) = item {
-            let listen_address = &c.listen_address;
-            writeln!(env, "RW_FRONTEND_LISTEN_ADDRESS=\"{listen_address}\"",).unwrap();
-            let port = &c.port;
-            writeln!(env, "RW_FRONTEND_PORT=\"{port}\"",).unwrap();
-            break;
+            _ => {}
         }
     }
     env