From 605eb34fe49041bbdbde47802cf445ffc5b5f33f Mon Sep 17 00:00:00 2001 From: xxchan Date: Mon, 14 Oct 2024 16:39:01 +0800 Subject: [PATCH] move --- .gitattributes | 2 +- ci/scripts/deterministic-e2e-test.sh | 4 ++-- ci/scripts/deterministic-recovery-test.sh | 2 +- ci/scripts/e2e-source-test.sh | 4 ++-- .../source_inline/kafka/avro/name_strategy.slt | 6 +++--- e2e_test/source_legacy/README.md | 4 ++-- e2e_test/source_legacy/basic/scripts/README.md | 12 ++++++++++++ .../basic/scripts}/prepare_ci_kafka.sh | 15 +++++++-------- .../basic/scripts}/schema_registry_producer.py | 0 .../source_legacy/basic/scripts}/test_data.zip | Bin .../scripts}/test_data/avro_complex_schema_bin.1 | Bin .../scripts}/test_data/avro_simple_schema_bin.1 | Bin .../basic/scripts}/test_data/bug_bash.1 | 0 .../scripts}/test_data/canal_json_double_field.1 | 0 .../basic/scripts}/test_data/cannal_json.1 | 0 .../test_data/debezium_compact_avro_json.1 | 0 .../test_data/debezium_ignore_case_json.1 | 0 .../basic/scripts}/test_data/debezium_log.1 | 0 .../test_data/debezium_log_no_schema_field.1 | 0 .../basic/scripts}/test_data/debezium_mess_key.1 | 0 .../test_data/debezium_mongo_json_customers.1 | 0 ...bezium_mongo_json_customers_no_schema_field.1 | 0 .../test_data/debezium_non_compact_avro_json.1 | 0 .../basic/scripts}/test_data/json_bytea.1 | 0 .../basic/scripts}/test_data/json_c.1 | 0 .../test_data/json_schema_without_schema.1 | 0 .../test_data/json_timestamptz_handling_mode.1 | 0 .../basic/scripts}/test_data/kafka_1_csv_topic.1 | 0 .../test_data/kafka_1_partition_mv_topic.1 | 0 .../scripts}/test_data/kafka_1_partition_topic.1 | 0 .../scripts}/test_data/kafka_2_partition_topic.2 | 0 .../scripts}/test_data/kafka_3_partition_topic.3 | 0 .../scripts}/test_data/kafka_4_partition_topic.4 | 0 .../kafka_4_partition_topic_generated_columns.1 | 0 .../kafka_4_partition_topic_with_100_message.4 | 0 .../basic/scripts}/test_data/kafka_json_schema.1 | 0 .../test_data/kafka_source_format_bytes.1 | 0 .../test_data/kafka_upsert_json_schema.1 | 0 .../basic/scripts}/test_data/maxwell_json.1 | 0 .../basic/scripts}/test_data/proto_c_bin.1 | 0 .../basic/scripts}/test_data/sink_debezium.1 | 0 .../basic/scripts}/test_data/sink_target.1 | 0 .../basic/scripts}/test_data/upsert_json.1 | 0 .../test_data/upsert_student_avro_json.1 | 0 ...t_student_key_not_subset_of_value_avro_json.1 | 0 .../basic/scripts}/test_data/weiling.1 | 0 scripts/source/README.md | 2 +- .../codec/tests/integration_tests/avro.rs | 2 +- 48 files changed, 32 insertions(+), 21 deletions(-) create mode 100644 e2e_test/source_legacy/basic/scripts/README.md rename {scripts/source => e2e_test/source_legacy/basic/scripts}/prepare_ci_kafka.sh (76%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/schema_registry_producer.py (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data.zip (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/avro_complex_schema_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/avro_simple_schema_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/bug_bash.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/canal_json_double_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/cannal_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_compact_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_ignore_case_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_log.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_log_no_schema_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_mess_key.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_mongo_json_customers.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_mongo_json_customers_no_schema_field.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/debezium_non_compact_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_bytea.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_c.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_schema_without_schema.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/json_timestamptz_handling_mode.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_1_csv_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_1_partition_mv_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_1_partition_topic.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_2_partition_topic.2 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_3_partition_topic.3 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_4_partition_topic.4 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_4_partition_topic_generated_columns.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_4_partition_topic_with_100_message.4 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_json_schema.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_source_format_bytes.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/kafka_upsert_json_schema.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/maxwell_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/proto_c_bin.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/sink_debezium.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/sink_target.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/upsert_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/upsert_student_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/upsert_student_key_not_subset_of_value_avro_json.1 (100%) rename {scripts/source => e2e_test/source_legacy/basic/scripts}/test_data/weiling.1 (100%) diff --git a/.gitattributes b/.gitattributes index becfd0ec71edf..64e8a463c51f6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,6 +1,6 @@ # regression test data src/tests/regress/data/** linguist-vendored # source test data -scripts/source/test_data/** linguist-vendored +e2e_test/source_legacy/basic/scripts/test_data/** linguist-vendored # generated grafana dashboard grafana/risingwave-dashboard.json linguist-generated diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh index e7c7b21310939..b0965688003c2 100755 --- a/ci/scripts/deterministic-e2e-test.sh +++ b/ci/scripts/deterministic-e2e-test.sh @@ -11,7 +11,7 @@ download-and-decompress-artifact risingwave_simulation . chmod +x ./risingwave_simulation echo "--- Extract data for Kafka" -pushd ./scripts/source/ +pushd ./e2e_test/source_legacy/basic/scripts/ mkdir -p ./test_data unzip -o test_data.zip -d . popd @@ -39,7 +39,7 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, batch" seq "$TEST_NUM" | parallel './risingwave_simulation ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/batch-{}.log && rm $LOGDIR/batch-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, kafka source" -seq "$TEST_NUM" | parallel './risingwave_simulation --kafka-datadir=./scripts/source/test_data ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log' +seq "$TEST_NUM" | parallel './risingwave_simulation --kafka-datadir=./e2e_test/source_legacy/basic/scripts/test_data ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, streaming" seq "$TEST_NUM" | parallel './risingwave_simulation -j 16 ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/parallel-streaming-{}.log && rm $LOGDIR/parallel-streaming-{}.log' diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 62d2250af96ec..bf1f28861c24a 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -93,6 +93,6 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, kafka source seq "$TEST_NUM" | parallel './risingwave_simulation \ --kill \ --kill-rate=${KILL_RATE} \ ---kafka-datadir=./scripts/source/test_data \ +--kafka-datadir=./e2e_test/source_legacy/basic/scripts/test_data \ ${EXTRA_ARGS:-} \ ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log' diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 5c53f84c24858..9f261f7b4cf08 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -148,11 +148,11 @@ echo "--- Kill cluster" risedev ci-kill export RISINGWAVE_CI=true -echo "--- e2e, ci-kafka-plus-pubsub, kafka and pubsub source" +echo "--- e2e, ci-kafka-plus-pubsub, legacy kafka tests" export RUST_MIN_STACK=4194304 RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ risedev ci-start ci-kafka -./scripts/source/prepare_ci_kafka.sh +./e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh risedev slt './e2e_test/source_legacy/basic/*.slt' risedev slt './e2e_test/source_legacy/basic/old_row_format_syntax/*.slt' diff --git a/e2e_test/source_inline/kafka/avro/name_strategy.slt b/e2e_test/source_inline/kafka/avro/name_strategy.slt index 212c79e6f31a8..f0b49a36ea284 100644 --- a/e2e_test/source_inline/kafka/avro/name_strategy.slt +++ b/e2e_test/source_inline/kafka/avro/name_strategy.slt @@ -25,7 +25,7 @@ create source s1 () with ( # Currently we are abusing this test case to also test data types. system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic" "avro" +python3 e2e_test/source_legacy/basic/scripts/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic" "avro" statement ok CREATE TABLE t_topic ( primary key (rw_key) ) @@ -44,7 +44,7 @@ FORMAT UPSERT ENCODE AVRO (schema.registry = '${RISEDEV_SCHEMA_REGISTRY_URL}'); ## topic: upsert_avro_json-record, key subject: string, value subject: CPLM.OBJ_ATTRIBUTE_VALUE system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "record" "avro" +python3 e2e_test/source_legacy/basic/scripts/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "record" "avro" statement error key\.message @@ -80,7 +80,7 @@ create table t_record_format_plain () with ( ## key subject: upsert_avro_json-topic-record-string ## value subject: upsert_avro_json-topic-record-CPLM.OBJ_ATTRIBUTE_VALUE system ok -python3 scripts/source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic-record" "avro" +python3 e2e_test/source_legacy/basic/scripts/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" e2e_test/source_inline/kafka/avro/upsert_avro_json "topic-record" "avro" diff --git a/e2e_test/source_legacy/README.md b/e2e_test/source_legacy/README.md index 611619cb13423..0b2d817df988c 100644 --- a/e2e_test/source_legacy/README.md +++ b/e2e_test/source_legacy/README.md @@ -7,8 +7,8 @@ Test in this directory needs some prior setup. -See also `ci/scripts/e2e-source-test.sh`, and `scripts/source` +See also `ci/scripts/e2e-source-test.sh`, and `e2e_test/source_legacy/basic/scripts` ## Kafka -`scripts/source/test_data` contains the data. Filename's convention is `.`. +`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `.`. diff --git a/e2e_test/source_legacy/basic/scripts/README.md b/e2e_test/source_legacy/basic/scripts/README.md new file mode 100644 index 0000000000000..928394a43007e --- /dev/null +++ b/e2e_test/source_legacy/basic/scripts/README.md @@ -0,0 +1,12 @@ +This folder contains scripts to prepare data for testing sources. + +## Kafka + +`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `.`. + +- If `` ends with `bin`, the whole file is a message with binary data. +- If `` ends with `avro_json` or `json_schema`: + - The first line is the schema. Key and value are separated by `^`. + - The rest of the lines are messages in JSON format. Key and value are separated by `^`. + - Produced to Kafka with `schema_registry_producer.py` (serialized to Avro or JSON) +- Otherwise, each line is a message, and key/value is separated by `^`. diff --git a/scripts/source/prepare_ci_kafka.sh b/e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh similarity index 76% rename from scripts/source/prepare_ci_kafka.sh rename to e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh index 27c88e6e63af2..75abd7095a925 100755 --- a/scripts/source/prepare_ci_kafka.sh +++ b/e2e_test/source_legacy/basic/scripts/prepare_ci_kafka.sh @@ -2,18 +2,19 @@ # Exits as soon as any line fails. set -e +export CARGO_MAKE_PRINT_TIME_SUMMARY=false SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)" -cd "$SCRIPT_PATH/.." || exit 1 -# cwd is /scripts +# SCRIPT_PATH is e2e_test/source_legacy/kafka/script/ +# cwd is e2e_test/source_legacy/kafka/ echo "$SCRIPT_PATH" +cd "$SCRIPT_PATH" -source ../.risingwave/config/risedev-env +source ../../../../.risingwave/config/risedev-env if [ "$1" == "compress" ]; then echo "Compress test_data/ into test_data.zip" - cd ./source zip_file=test_data.zip if [ -f "$zip_file" ]; then rm "$zip_file" @@ -23,10 +24,8 @@ if [ "$1" == "compress" ]; then fi echo "--- Extract data for Kafka" -cd ./source/ mkdir -p ./test_data/ch_benchmark/ unzip -o test_data.zip -d . -cd .. echo "path:${SCRIPT_PATH}/test_data/**/*" @@ -58,9 +57,9 @@ for filename in $kafka_data_files; do if [[ "$topic" = *bin ]]; then kcat -P -b "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" -t "$topic" "$filename" elif [[ "$topic" = *avro_json ]]; then - python3 source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "avro" + python3 schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "avro" elif [[ "$topic" = *json_schema ]]; then - python3 source/schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "json" + python3 schema_registry_producer.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "$filename" "topic" "json" else cat "$filename" | kcat -P -K ^ -b "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" -t "$topic" fi diff --git a/scripts/source/schema_registry_producer.py b/e2e_test/source_legacy/basic/scripts/schema_registry_producer.py similarity index 100% rename from scripts/source/schema_registry_producer.py rename to e2e_test/source_legacy/basic/scripts/schema_registry_producer.py diff --git a/scripts/source/test_data.zip b/e2e_test/source_legacy/basic/scripts/test_data.zip similarity index 100% rename from scripts/source/test_data.zip rename to e2e_test/source_legacy/basic/scripts/test_data.zip diff --git a/scripts/source/test_data/avro_complex_schema_bin.1 b/e2e_test/source_legacy/basic/scripts/test_data/avro_complex_schema_bin.1 similarity index 100% rename from scripts/source/test_data/avro_complex_schema_bin.1 rename to e2e_test/source_legacy/basic/scripts/test_data/avro_complex_schema_bin.1 diff --git a/scripts/source/test_data/avro_simple_schema_bin.1 b/e2e_test/source_legacy/basic/scripts/test_data/avro_simple_schema_bin.1 similarity index 100% rename from scripts/source/test_data/avro_simple_schema_bin.1 rename to e2e_test/source_legacy/basic/scripts/test_data/avro_simple_schema_bin.1 diff --git a/scripts/source/test_data/bug_bash.1 b/e2e_test/source_legacy/basic/scripts/test_data/bug_bash.1 similarity index 100% rename from scripts/source/test_data/bug_bash.1 rename to e2e_test/source_legacy/basic/scripts/test_data/bug_bash.1 diff --git a/scripts/source/test_data/canal_json_double_field.1 b/e2e_test/source_legacy/basic/scripts/test_data/canal_json_double_field.1 similarity index 100% rename from scripts/source/test_data/canal_json_double_field.1 rename to e2e_test/source_legacy/basic/scripts/test_data/canal_json_double_field.1 diff --git a/scripts/source/test_data/cannal_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/cannal_json.1 similarity index 100% rename from scripts/source/test_data/cannal_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/cannal_json.1 diff --git a/scripts/source/test_data/debezium_compact_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_compact_avro_json.1 similarity index 100% rename from scripts/source/test_data/debezium_compact_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_compact_avro_json.1 diff --git a/scripts/source/test_data/debezium_ignore_case_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_ignore_case_json.1 similarity index 100% rename from scripts/source/test_data/debezium_ignore_case_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_ignore_case_json.1 diff --git a/scripts/source/test_data/debezium_log.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_log.1 similarity index 100% rename from scripts/source/test_data/debezium_log.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_log.1 diff --git a/scripts/source/test_data/debezium_log_no_schema_field.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_log_no_schema_field.1 similarity index 100% rename from scripts/source/test_data/debezium_log_no_schema_field.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_log_no_schema_field.1 diff --git a/scripts/source/test_data/debezium_mess_key.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_mess_key.1 similarity index 100% rename from scripts/source/test_data/debezium_mess_key.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_mess_key.1 diff --git a/scripts/source/test_data/debezium_mongo_json_customers.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers.1 similarity index 100% rename from scripts/source/test_data/debezium_mongo_json_customers.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers.1 diff --git a/scripts/source/test_data/debezium_mongo_json_customers_no_schema_field.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers_no_schema_field.1 similarity index 100% rename from scripts/source/test_data/debezium_mongo_json_customers_no_schema_field.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_mongo_json_customers_no_schema_field.1 diff --git a/scripts/source/test_data/debezium_non_compact_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/debezium_non_compact_avro_json.1 similarity index 100% rename from scripts/source/test_data/debezium_non_compact_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/debezium_non_compact_avro_json.1 diff --git a/scripts/source/test_data/json_bytea.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_bytea.1 similarity index 100% rename from scripts/source/test_data/json_bytea.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_bytea.1 diff --git a/scripts/source/test_data/json_c.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_c.1 similarity index 100% rename from scripts/source/test_data/json_c.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_c.1 diff --git a/scripts/source/test_data/json_schema_without_schema.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_schema_without_schema.1 similarity index 100% rename from scripts/source/test_data/json_schema_without_schema.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_schema_without_schema.1 diff --git a/scripts/source/test_data/json_timestamptz_handling_mode.1 b/e2e_test/source_legacy/basic/scripts/test_data/json_timestamptz_handling_mode.1 similarity index 100% rename from scripts/source/test_data/json_timestamptz_handling_mode.1 rename to e2e_test/source_legacy/basic/scripts/test_data/json_timestamptz_handling_mode.1 diff --git a/scripts/source/test_data/kafka_1_csv_topic.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_1_csv_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_csv_topic.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_1_csv_topic.1 diff --git a/scripts/source/test_data/kafka_1_partition_mv_topic.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_mv_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_partition_mv_topic.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_mv_topic.1 diff --git a/scripts/source/test_data/kafka_1_partition_topic.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_topic.1 similarity index 100% rename from scripts/source/test_data/kafka_1_partition_topic.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_1_partition_topic.1 diff --git a/scripts/source/test_data/kafka_2_partition_topic.2 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_2_partition_topic.2 similarity index 100% rename from scripts/source/test_data/kafka_2_partition_topic.2 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_2_partition_topic.2 diff --git a/scripts/source/test_data/kafka_3_partition_topic.3 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_3_partition_topic.3 similarity index 100% rename from scripts/source/test_data/kafka_3_partition_topic.3 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_3_partition_topic.3 diff --git a/scripts/source/test_data/kafka_4_partition_topic.4 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic.4 similarity index 100% rename from scripts/source/test_data/kafka_4_partition_topic.4 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic.4 diff --git a/scripts/source/test_data/kafka_4_partition_topic_generated_columns.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_generated_columns.1 similarity index 100% rename from scripts/source/test_data/kafka_4_partition_topic_generated_columns.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_generated_columns.1 diff --git a/scripts/source/test_data/kafka_4_partition_topic_with_100_message.4 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_with_100_message.4 similarity index 100% rename from scripts/source/test_data/kafka_4_partition_topic_with_100_message.4 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_4_partition_topic_with_100_message.4 diff --git a/scripts/source/test_data/kafka_json_schema.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_json_schema.1 similarity index 100% rename from scripts/source/test_data/kafka_json_schema.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_json_schema.1 diff --git a/scripts/source/test_data/kafka_source_format_bytes.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_source_format_bytes.1 similarity index 100% rename from scripts/source/test_data/kafka_source_format_bytes.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_source_format_bytes.1 diff --git a/scripts/source/test_data/kafka_upsert_json_schema.1 b/e2e_test/source_legacy/basic/scripts/test_data/kafka_upsert_json_schema.1 similarity index 100% rename from scripts/source/test_data/kafka_upsert_json_schema.1 rename to e2e_test/source_legacy/basic/scripts/test_data/kafka_upsert_json_schema.1 diff --git a/scripts/source/test_data/maxwell_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/maxwell_json.1 similarity index 100% rename from scripts/source/test_data/maxwell_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/maxwell_json.1 diff --git a/scripts/source/test_data/proto_c_bin.1 b/e2e_test/source_legacy/basic/scripts/test_data/proto_c_bin.1 similarity index 100% rename from scripts/source/test_data/proto_c_bin.1 rename to e2e_test/source_legacy/basic/scripts/test_data/proto_c_bin.1 diff --git a/scripts/source/test_data/sink_debezium.1 b/e2e_test/source_legacy/basic/scripts/test_data/sink_debezium.1 similarity index 100% rename from scripts/source/test_data/sink_debezium.1 rename to e2e_test/source_legacy/basic/scripts/test_data/sink_debezium.1 diff --git a/scripts/source/test_data/sink_target.1 b/e2e_test/source_legacy/basic/scripts/test_data/sink_target.1 similarity index 100% rename from scripts/source/test_data/sink_target.1 rename to e2e_test/source_legacy/basic/scripts/test_data/sink_target.1 diff --git a/scripts/source/test_data/upsert_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/upsert_json.1 similarity index 100% rename from scripts/source/test_data/upsert_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/upsert_json.1 diff --git a/scripts/source/test_data/upsert_student_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/upsert_student_avro_json.1 similarity index 100% rename from scripts/source/test_data/upsert_student_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/upsert_student_avro_json.1 diff --git a/scripts/source/test_data/upsert_student_key_not_subset_of_value_avro_json.1 b/e2e_test/source_legacy/basic/scripts/test_data/upsert_student_key_not_subset_of_value_avro_json.1 similarity index 100% rename from scripts/source/test_data/upsert_student_key_not_subset_of_value_avro_json.1 rename to e2e_test/source_legacy/basic/scripts/test_data/upsert_student_key_not_subset_of_value_avro_json.1 diff --git a/scripts/source/test_data/weiling.1 b/e2e_test/source_legacy/basic/scripts/test_data/weiling.1 similarity index 100% rename from scripts/source/test_data/weiling.1 rename to e2e_test/source_legacy/basic/scripts/test_data/weiling.1 diff --git a/scripts/source/README.md b/scripts/source/README.md index 311c058a8230b..928394a43007e 100644 --- a/scripts/source/README.md +++ b/scripts/source/README.md @@ -2,7 +2,7 @@ This folder contains scripts to prepare data for testing sources. ## Kafka -`scripts/source/test_data` contains the data. Filename's convention is `.`. +`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `.`. - If `` ends with `bin`, the whole file is a message with binary data. - If `` ends with `avro_json` or `json_schema`: diff --git a/src/connector/codec/tests/integration_tests/avro.rs b/src/connector/codec/tests/integration_tests/avro.rs index ab1df6e7e82b8..d916dc1aba426 100644 --- a/src/connector/codec/tests/integration_tests/avro.rs +++ b/src/connector/codec/tests/integration_tests/avro.rs @@ -119,7 +119,7 @@ fn check( expected_risingwave_data.assert_eq(&format!("{}", data_str.iter().format("\n----\n"))); } -// This corresponds to legacy `scripts/source/test_data/avro_simple_schema_bin.1`. TODO: remove that file. +// This corresponds to legacy `e2e_test/source_legacy/basic/scripts/test_data/avro_simple_schema_bin.1`. TODO: remove that file. #[test] fn test_simple() { check(