move

risingwavelabs · Oct 14, 2024 · a251518 · a251518
1 parent 99c88c4
commit a251518
Show file tree

Hide file tree

Showing 41 changed files with 18 additions and 6 deletions.
diff --git a/e2e_test/source_legacy/basic/scripts/README.md b/e2e_test/source_legacy/basic/scripts/README.md
@@ -0,0 +1,12 @@
+This folder contains scripts to prepare data for testing sources.
+
+## Kafka
+
+`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `<topic_name>.<n_partitions>`.
+
+- If `<topic_name>` ends with `bin`, the whole file is a message with binary data.
+- If `<topic_name>` ends with `avro_json` or `json_schema`:
+    - The first line is the schema. Key and value are separated by `^`.
+    - The rest of the lines are messages in JSON format. Key and value are separated by `^`.
+    - Produced to Kafka with `schema_registry_producer.py` (serialized to Avro or JSON)
+- Otherwise, each line is a message, and key/value is separated by `^`.
diff --git a/scripts/source/prepare_ci_kafka.sh → ..._legacy/basic/scripts/prepare_ci_kafka.sh b/scripts/source/prepare_ci_kafka.sh → ..._legacy/basic/scripts/prepare_ci_kafka.sh
@@ -4,16 +4,16 @@
 set -e
 
 SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
-cd "$SCRIPT_PATH/.." || exit 1
-# cwd is /scripts
+# SCRIPT_PATH is e2e_test/source_legacy/kafka/script/
+# cwd is e2e_test/source_legacy/kafka/
 
 echo "$SCRIPT_PATH"
 
-source ../.risingwave/config/risedev-env
+source ../../../.risingwave/config/risedev-env
 
 if [ "$1" == "compress" ]; then
   echo "Compress test_data/ into test_data.zip"
-  cd ./source
+  cd ./script
   zip_file=test_data.zip
   if [ -f "$zip_file" ]; then
     rm "$zip_file"
@@ -23,7 +23,7 @@ if [ "$1" == "compress" ]; then
 fi
 
 echo "--- Extract data for Kafka"
-cd ./source/
+cd ./script/
 mkdir -p ./test_data/ch_benchmark/
 unzip -o test_data.zip -d .
 cd ..

diff --git a/scripts/source/schema_registry_producer.py → ...basic/scripts/schema_registry_producer.py b/scripts/source/schema_registry_producer.py → ...basic/scripts/schema_registry_producer.py
diff --git a/scripts/source/test_data.zip → ...source_legacy/basic/scripts/test_data.zip b/scripts/source/test_data.zip → ...source_legacy/basic/scripts/test_data.zip
diff --git a/...ource/test_data/avro_complex_schema_bin.1 → ...ripts/test_data/avro_complex_schema_bin.1 b/...ource/test_data/avro_complex_schema_bin.1 → ...ripts/test_data/avro_complex_schema_bin.1
diff --git a/...source/test_data/avro_simple_schema_bin.1 → ...cripts/test_data/avro_simple_schema_bin.1 b/...source/test_data/avro_simple_schema_bin.1 → ...cripts/test_data/avro_simple_schema_bin.1
diff --git a/scripts/source/test_data/bug_bash.1 → ...legacy/basic/scripts/test_data/bug_bash.1 b/scripts/source/test_data/bug_bash.1 → ...legacy/basic/scripts/test_data/bug_bash.1
diff --git a/...ource/test_data/canal_json_double_field.1 → ...ripts/test_data/canal_json_double_field.1 b/...ource/test_data/canal_json_double_field.1 → ...ripts/test_data/canal_json_double_field.1
diff --git a/scripts/source/test_data/cannal_json.1 → ...acy/basic/scripts/test_data/cannal_json.1 b/scripts/source/test_data/cannal_json.1 → ...acy/basic/scripts/test_data/cannal_json.1
diff --git a/...ce/test_data/debezium_compact_avro_json.1 → ...ts/test_data/debezium_compact_avro_json.1 b/...ce/test_data/debezium_compact_avro_json.1 → ...ts/test_data/debezium_compact_avro_json.1
diff --git a/...rce/test_data/debezium_ignore_case_json.1 → ...pts/test_data/debezium_ignore_case_json.1 b/...rce/test_data/debezium_ignore_case_json.1 → ...pts/test_data/debezium_ignore_case_json.1
diff --git a/scripts/source/test_data/debezium_log.1 → ...cy/basic/scripts/test_data/debezium_log.1 b/scripts/source/test_data/debezium_log.1 → ...cy/basic/scripts/test_data/debezium_log.1
diff --git a/.../test_data/debezium_log_no_schema_field.1 → .../test_data/debezium_log_no_schema_field.1 b/.../test_data/debezium_log_no_schema_field.1 → .../test_data/debezium_log_no_schema_field.1
diff --git a/scripts/source/test_data/debezium_mess_key.1 → ...sic/scripts/test_data/debezium_mess_key.1 b/scripts/source/test_data/debezium_mess_key.1 → ...sic/scripts/test_data/debezium_mess_key.1
diff --git a/...test_data/debezium_mongo_json_customers.1 → ...test_data/debezium_mongo_json_customers.1 b/...test_data/debezium_mongo_json_customers.1 → ...test_data/debezium_mongo_json_customers.1
diff --git a/...um_mongo_json_customers_no_schema_field.1 → ...um_mongo_json_customers_no_schema_field.1 b/...um_mongo_json_customers_no_schema_field.1 → ...um_mongo_json_customers_no_schema_field.1
diff --git a/...est_data/debezium_non_compact_avro_json.1 → ...est_data/debezium_non_compact_avro_json.1 b/...est_data/debezium_non_compact_avro_json.1 → ...est_data/debezium_non_compact_avro_json.1
diff --git a/scripts/source/test_data/json_bytea.1 → ...gacy/basic/scripts/test_data/json_bytea.1 b/scripts/source/test_data/json_bytea.1 → ...gacy/basic/scripts/test_data/json_bytea.1
diff --git a/scripts/source/test_data/json_c.1 → ...e_legacy/basic/scripts/test_data/json_c.1 b/scripts/source/test_data/json_c.1 → ...e_legacy/basic/scripts/test_data/json_c.1
diff --git a/...ce/test_data/json_schema_without_schema.1 → ...ts/test_data/json_schema_without_schema.1 b/...ce/test_data/json_schema_without_schema.1 → ...ts/test_data/json_schema_without_schema.1
diff --git a/...est_data/json_timestamptz_handling_mode.1 → ...est_data/json_timestamptz_handling_mode.1 b/...est_data/json_timestamptz_handling_mode.1 → ...est_data/json_timestamptz_handling_mode.1
diff --git a/scripts/source/test_data/kafka_1_csv_topic.1 → ...sic/scripts/test_data/kafka_1_csv_topic.1 b/scripts/source/test_data/kafka_1_csv_topic.1 → ...sic/scripts/test_data/kafka_1_csv_topic.1
diff --git a/...ce/test_data/kafka_1_partition_mv_topic.1 → ...ts/test_data/kafka_1_partition_mv_topic.1 b/...ce/test_data/kafka_1_partition_mv_topic.1 → ...ts/test_data/kafka_1_partition_mv_topic.1
diff --git a/...ource/test_data/kafka_1_partition_topic.1 → ...ripts/test_data/kafka_1_partition_topic.1 b/...ource/test_data/kafka_1_partition_topic.1 → ...ripts/test_data/kafka_1_partition_topic.1
diff --git a/...ource/test_data/kafka_2_partition_topic.2 → ...ripts/test_data/kafka_2_partition_topic.2 b/...ource/test_data/kafka_2_partition_topic.2 → ...ripts/test_data/kafka_2_partition_topic.2
diff --git a/...ource/test_data/kafka_3_partition_topic.3 → ...ripts/test_data/kafka_3_partition_topic.3 b/...ource/test_data/kafka_3_partition_topic.3 → ...ripts/test_data/kafka_3_partition_topic.3
diff --git a/...ource/test_data/kafka_4_partition_topic.4 → ...ripts/test_data/kafka_4_partition_topic.4 b/...ource/test_data/kafka_4_partition_topic.4 → ...ripts/test_data/kafka_4_partition_topic.4
diff --git a/...fka_4_partition_topic_generated_columns.1 → ...fka_4_partition_topic_generated_columns.1 b/...fka_4_partition_topic_generated_columns.1 → ...fka_4_partition_topic_generated_columns.1
diff --git a/...afka_4_partition_topic_with_100_message.4 → ...afka_4_partition_topic_with_100_message.4 b/...afka_4_partition_topic_with_100_message.4 → ...afka_4_partition_topic_with_100_message.4
diff --git a/scripts/source/test_data/kafka_json_schema.1 → ...sic/scripts/test_data/kafka_json_schema.1 b/scripts/source/test_data/kafka_json_schema.1 → ...sic/scripts/test_data/kafka_json_schema.1
diff --git a/...rce/test_data/kafka_source_format_bytes.1 → ...pts/test_data/kafka_source_format_bytes.1 b/...rce/test_data/kafka_source_format_bytes.1 → ...pts/test_data/kafka_source_format_bytes.1
diff --git a/...urce/test_data/kafka_upsert_json_schema.1 → ...ipts/test_data/kafka_upsert_json_schema.1 b/...urce/test_data/kafka_upsert_json_schema.1 → ...ipts/test_data/kafka_upsert_json_schema.1
diff --git a/scripts/source/test_data/maxwell_json.1 → ...cy/basic/scripts/test_data/maxwell_json.1 b/scripts/source/test_data/maxwell_json.1 → ...cy/basic/scripts/test_data/maxwell_json.1
diff --git a/scripts/source/test_data/proto_c_bin.1 → ...acy/basic/scripts/test_data/proto_c_bin.1 b/scripts/source/test_data/proto_c_bin.1 → ...acy/basic/scripts/test_data/proto_c_bin.1
diff --git a/scripts/source/test_data/sink_debezium.1 → ...y/basic/scripts/test_data/sink_debezium.1 b/scripts/source/test_data/sink_debezium.1 → ...y/basic/scripts/test_data/sink_debezium.1
diff --git a/scripts/source/test_data/sink_target.1 → ...acy/basic/scripts/test_data/sink_target.1 b/scripts/source/test_data/sink_target.1 → ...acy/basic/scripts/test_data/sink_target.1
diff --git a/scripts/source/test_data/upsert_json.1 → ...acy/basic/scripts/test_data/upsert_json.1 b/scripts/source/test_data/upsert_json.1 → ...acy/basic/scripts/test_data/upsert_json.1
diff --git a/...urce/test_data/upsert_student_avro_json.1 → ...ipts/test_data/upsert_student_avro_json.1 b/...urce/test_data/upsert_student_avro_json.1 → ...ipts/test_data/upsert_student_avro_json.1
diff --git a/...udent_key_not_subset_of_value_avro_json.1 → ...udent_key_not_subset_of_value_avro_json.1 b/...udent_key_not_subset_of_value_avro_json.1 → ...udent_key_not_subset_of_value_avro_json.1
diff --git a/scripts/source/test_data/weiling.1 → ..._legacy/basic/scripts/test_data/weiling.1 b/scripts/source/test_data/weiling.1 → ..._legacy/basic/scripts/test_data/weiling.1
diff --git a/scripts/source/README.md b/scripts/source/README.md
@@ -2,7 +2,7 @@ This folder contains scripts to prepare data for testing sources.
 
 ## Kafka
 
-`scripts/source/test_data` contains the data. Filename's convention is `<topic_name>.<n_partitions>`.
+`e2e_test/source_legacy/basic/scripts/test_data` contains the data. Filename's convention is `<topic_name>.<n_partitions>`.
 
 - If `<topic_name>` ends with `bin`, the whole file is a message with binary data.
 - If `<topic_name>` ends with `avro_json` or `json_schema`: