From c3c6878c0ed8002d352bbce39f3278a6bf2b2087 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 6 Sep 2023 05:09:03 +0000 Subject: [PATCH 01/21] feat(sink): sink with schema --- ci/scripts/e2e-kafka-sink-test.sh | 28 ++++++ e2e_test/sink/kafka/create_sink.slt | 10 ++ e2e_test/sink/kafka/upsert_schema1.result | 10 ++ e2e_test/sink/kafka/upsert_schema2.result | 11 +++ e2e_test/sink/kafka/upsert_schema3.result | 12 +++ src/connector/src/sink/kafka.rs | 9 ++ src/connector/src/sink/kinesis.rs | 2 + src/connector/src/sink/utils.rs | 114 +++++++++++++++++++--- 8 files changed, 185 insertions(+), 11 deletions(-) create mode 100644 e2e_test/sink/kafka/upsert_schema1.result create mode 100644 e2e_test/sink/kafka/upsert_schema2.result create mode 100644 e2e_test/sink/kafka/upsert_schema3.result diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh index 7f03945fe5b6..32d801110420 100755 --- a/ci/scripts/e2e-kafka-sink-test.sh +++ b/ci/scripts/e2e-kafka-sink-test.sh @@ -5,6 +5,7 @@ set -euo pipefail ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-append-only --create > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert --create > /dev/null 2>&1 +./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --create > /dev/null 2>&1 ./.risingwave/bin/kafka/bin/kafka-topics.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --create > /dev/null 2>&1 sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/create_sink.slt' @@ -28,6 +29,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema +echo "testing upsert kafka sink with schema" +diff ./e2e_test/sink/kafka/upsert_schema1.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 10 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink echo "testing debezium kafka sink" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 10 | sort) > ./e2e_test/sink/kafka/debezium1.tmp.result 2> /dev/null @@ -62,6 +72,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema after update +echo "testing upsert kafka sink with schema" +diff ./e2e_test/sink/kafka/upsert_schema2.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after update echo "testing debezium kafka sink after updating data" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 11 | sort) > ./e2e_test/sink/kafka/debezium2.tmp.result 2> /dev/null @@ -87,6 +106,15 @@ if [ $? -ne 0 ]; then exit 1 fi +# test upsert kafka sink with schema after delete +echo "testing upsert kafka sink with schema" +diff ./e2e_test/sink/kafka/upsert_schema3.result \ +<((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null) +if [ $? -ne 0 ]; then + echo "The output for upsert sink with schema is not as expected." + exit 1 +fi + # test debezium kafka sink after delete echo "testing debezium kafka sink after deleting data" (./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-debezium --property print.key=true --from-beginning --max-messages 13 | sort) > ./e2e_test/sink/kafka/debezium3.tmp.result 2> /dev/null diff --git a/e2e_test/sink/kafka/create_sink.slt b/e2e_test/sink/kafka/create_sink.slt index b1b48ce93bea..4cbc1a2e8726 100644 --- a/e2e_test/sink/kafka/create_sink.slt +++ b/e2e_test/sink/kafka/create_sink.slt @@ -106,6 +106,16 @@ create sink si_kafka_upsert from t_kafka with ( primary_key = 'id', ); +statement ok +create sink si_kafka_upsert_schema from t_kafka with ( + connector = 'kafka', + properties.bootstrap.server = '127.0.0.1:29092', + topic = 'test-rw-sink-upsert-schema', + type = 'upsert', + primary_key = 'id', + schema.enable = true, +); + statement ok create sink si_kafka_debezium from t_kafka with ( connector = 'kafka', diff --git a/e2e_test/sink/kafka/upsert_schema1.result b/e2e_test/sink/kafka/upsert_schema1.result new file mode 100644 index 000000000000..2d34a0d7ec0b --- /dev/null +++ b/e2e_test/sink/kafka/upsert_schema1.result @@ -0,0 +1,10 @@ +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} \ No newline at end of file diff --git a/e2e_test/sink/kafka/upsert_schema2.result b/e2e_test/sink/kafka/upsert_schema2.result new file mode 100644 index 000000000000..cf17ecc05af5 --- /dev/null +++ b/e2e_test/sink/kafka/upsert_schema2.result @@ -0,0 +1,11 @@ +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} \ No newline at end of file diff --git a/e2e_test/sink/kafka/upsert_schema3.result b/e2e_test/sink/kafka/upsert_schema3.result new file mode 100644 index 000000000000..5cc4d171a1ea --- /dev/null +++ b/e2e_test/sink/kafka/upsert_schema3.result @@ -0,0 +1,12 @@ +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} \ No newline at end of file diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs index 4c7fc317edd3..6a45c24e7d1c 100644 --- a/src/connector/src/sink/kafka.rs +++ b/src/connector/src/sink/kafka.rs @@ -203,6 +203,13 @@ pub struct KafkaConfig { )] pub use_transaction: bool, + #[serde( + rename = "schema.enable", + default, + deserialize_with = "deserialize_bool_from_string" + )] + pub schema_enable: bool, + /// We have parsed the primary key for an upsert kafka sink into a `usize` vector representing /// the indices of the pk columns in the frontend, so we simply store the primary key here /// as a string. @@ -496,6 +503,8 @@ impl KafkaSinkWriter { &self.schema, &self.pk_indices, chunk, + self.config.schema_enable, + Some(format!("{}.{}", self.db_name, self.sink_from_name)), UpsertAdapterOpts::default(), ); diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs index 4a6f3a789bf1..9619cef1a924 100644 --- a/src/connector/src/sink/kinesis.rs +++ b/src/connector/src/sink/kinesis.rs @@ -207,6 +207,8 @@ impl KinesisSinkWriter { &self.schema, &self.pk_indices, chunk, + false, + None, UpsertAdapterOpts::default(), ); diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index 1eb2f91e9af2..cf54d53f9280 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -399,32 +399,65 @@ fn datum_to_json_object( #[derive(Debug, Clone, Default)] pub struct UpsertAdapterOpts {} +fn gen_event_object( + schema: &Schema, + object: Value, + enable_schema: bool, + name: &Option, +) -> Option { + if enable_schema { + Some(json!({ + "schema": generate_json_converter_schema(&schema.fields, name.as_ref().unwrap()), + "payload": object, + })) + } else { + Some(object) + } +} + #[try_stream(ok = (Option, Option), error = SinkError)] pub async fn gen_upsert_message_stream<'a>( schema: &'a Schema, pk_indices: &'a [usize], chunk: StreamChunk, + enable_schema: bool, + schema_name: Option, _opts: UpsertAdapterOpts, ) { for (op, row) in chunk.rows() { - let event_key_object = Some(Value::Object(pk_to_json(row, &schema.fields, pk_indices)?)); + let event_key_object = gen_event_object( + schema, + Value::Object(pk_to_json(row, &schema.fields, pk_indices)?), + enable_schema, + &schema_name, + ); let event_object = match op { - Op::Insert => Some(Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?)), + Op::Insert => gen_event_object( + schema, + Value::Object(record_to_json( + row, + &schema.fields, + TimestampHandlingMode::Milli, + )?), + enable_schema, + &schema_name, + ), Op::Delete => Some(Value::Null), Op::UpdateDelete => { // upsert semantic does not require update delete event continue; } - Op::UpdateInsert => Some(Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?)), + Op::UpdateInsert => gen_event_object( + schema, + Value::Object(record_to_json( + row, + &schema.fields, + TimestampHandlingMode::Milli, + )?), + enable_schema, + &schema_name, + ), }; yield (event_key_object, event_object); @@ -456,6 +489,65 @@ pub async fn gen_append_only_message_stream<'a>( } } +// reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSchema.java#L39 +fn json_converter_field_to_json(field: &Field) -> Value { + let mut mapping = Map::with_capacity(3); + let type_mapping = |rw_type: &DataType| match rw_type { + DataType::Boolean => "boolean", + DataType::Int16 => "int16", + DataType::Int32 => "int32", + DataType::Int64 => "int64", + DataType::Float32 => "float", + DataType::Float64 => "double", + DataType::Decimal => "string", + DataType::Date => "int32", + DataType::Varchar => "string", + DataType::Time => "int64", + DataType::Timestamp => "int64", + DataType::Timestamptz => "string", + DataType::Interval => "string", + DataType::Struct(_) => "struct", + DataType::List(_) => "array", + DataType::Bytea => "bytes", + DataType::Jsonb => "string", + DataType::Serial => "int32", + DataType::Int256 => "string", + }; + mapping.insert("type".into(), json!(type_mapping(&field.data_type))); + mapping.insert("optional".into(), json!("true")); + match &field.data_type { + DataType::Struct(_) => { + let mut sub_fields = Vec::new(); + for sub_field in &field.sub_fields { + sub_fields.push(json_converter_field_to_json(sub_field)); + } + mapping.insert("fields".into(), json!(sub_fields)); + } + DataType::List(list_type) => { + mapping.insert( + "items".into(), + json!({ + "type": type_mapping(list_type), + }), + ); + } + _ => { + mapping.insert("field".into(), json!(field.name)); + } + } + json!(mapping) +} + +/// Generate schema for Kafka's `JsonConverter` when `schema.enable` is true +fn generate_json_converter_schema(fields: &[Field], name: &str) -> Value { + json!({ + "type": "struct", + "fields": fields.iter().map(json_converter_field_to_json).collect::>(), + "optional": "false", + "name": name, + }) +} + #[cfg(test)] mod tests { From 1bcc6e27b4b7ce4ef8643d64135f03fa2a58f01e Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 6 Sep 2023 05:25:31 +0000 Subject: [PATCH 02/21] fix test --- e2e_test/sink/kafka/upsert_schema1.result | 20 +++++++++---------- e2e_test/sink/kafka/upsert_schema2.result | 22 ++++++++++----------- e2e_test/sink/kafka/upsert_schema3.result | 24 +++++++++++------------ 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/e2e_test/sink/kafka/upsert_schema1.result b/e2e_test/sink/kafka/upsert_schema1.result index 2d34a0d7ec0b..44effd0cc56d 100644 --- a/e2e_test/sink/kafka/upsert_schema1.result +++ b/e2e_test/sink/kafka/upsert_schema1.result @@ -1,10 +1,10 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} \ No newline at end of file +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema2.result b/e2e_test/sink/kafka/upsert_schema2.result index cf17ecc05af5..f81299a9b99c 100644 --- a/e2e_test/sink/kafka/upsert_schema2.result +++ b/e2e_test/sink/kafka/upsert_schema2.result @@ -1,11 +1,11 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} \ No newline at end of file +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema3.result b/e2e_test/sink/kafka/upsert_schema3.result index 5cc4d171a1ea..90165cd8e79b 100644 --- a/e2e_test/sink/kafka/upsert_schema3.result +++ b/e2e_test/sink/kafka/upsert_schema3.result @@ -1,12 +1,12 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} \ No newline at end of file +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} From 0fc8af5bdb1a35fdfa1915374a2ddb3c7b17d8a7 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 6 Sep 2023 05:35:30 +0000 Subject: [PATCH 03/21] fix test --- e2e_test/sink/kafka/drop_sink.slt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e_test/sink/kafka/drop_sink.slt b/e2e_test/sink/kafka/drop_sink.slt index 4b9b10eceb3d..9479671b9f0f 100644 --- a/e2e_test/sink/kafka/drop_sink.slt +++ b/e2e_test/sink/kafka/drop_sink.slt @@ -7,5 +7,8 @@ drop sink si_kafka_upsert; statement ok drop sink si_kafka_debezium; +statement ok +drop sink si_kafka_upsert_schema; + statement ok drop table t_kafka; From 74f131e99386fdee33fea6ce18d28143fffb58a0 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 6 Sep 2023 05:47:52 +0000 Subject: [PATCH 04/21] chore --- ci/scripts/e2e-kafka-sink-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh index 32d801110420..54912f8d5c17 100755 --- a/ci/scripts/e2e-kafka-sink-test.sh +++ b/ci/scripts/e2e-kafka-sink-test.sh @@ -73,7 +73,7 @@ if [ $? -ne 0 ]; then fi # test upsert kafka sink with schema after update -echo "testing upsert kafka sink with schema" +echo "testing upsert kafka sink with schema after updating data" diff ./e2e_test/sink/kafka/upsert_schema2.result \ <((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 11 | sort) 2> /dev/null) if [ $? -ne 0 ]; then @@ -107,7 +107,7 @@ if [ $? -ne 0 ]; then fi # test upsert kafka sink with schema after delete -echo "testing upsert kafka sink with schema" +echo "testing upsert kafka sink with schema after deleting data" diff ./e2e_test/sink/kafka/upsert_schema3.result \ <((./.risingwave/bin/kafka/bin/kafka-console-consumer.sh --bootstrap-server 127.0.0.1:29092 --topic test-rw-sink-upsert-schema --from-beginning --property print.key=true --max-messages 12 | sort) 2> /dev/null) if [ $? -ne 0 ]; then From 303ff6796835abe09fa4e8bc4bdfb8fcc9b061f1 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 6 Sep 2023 06:41:15 +0000 Subject: [PATCH 05/21] add test --- src/connector/src/sink/utils.rs | 136 ++++++++++++++++++++++++++++++-- 1 file changed, 131 insertions(+), 5 deletions(-) diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index cf54d53f9280..3580a55cffb5 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -491,7 +491,7 @@ pub async fn gen_append_only_message_stream<'a>( // reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSchema.java#L39 fn json_converter_field_to_json(field: &Field) -> Value { - let mut mapping = Map::with_capacity(3); + let mut mapping = Map::with_capacity(4); let type_mapping = |rw_type: &DataType| match rw_type { DataType::Boolean => "boolean", DataType::Int16 => "int16", @@ -515,6 +515,7 @@ fn json_converter_field_to_json(field: &Field) -> Value { }; mapping.insert("type".into(), json!(type_mapping(&field.data_type))); mapping.insert("optional".into(), json!("true")); + mapping.insert("field".into(), json!(field.name)); match &field.data_type { DataType::Struct(_) => { let mut sub_fields = Vec::new(); @@ -531,9 +532,7 @@ fn json_converter_field_to_json(field: &Field) -> Value { }), ); } - _ => { - mapping.insert("field".into(), json!(field.name)); - } + _ => {} } json!(mapping) } @@ -551,7 +550,7 @@ fn generate_json_converter_schema(fields: &[Field], name: &str) -> Value { #[cfg(test)] mod tests { - use risingwave_common::types::{DataType, Interval, ScalarImpl, Time, Timestamp}; + use risingwave_common::types::{DataType, Interval, ScalarImpl, StructType, Time, Timestamp}; use super::*; #[test] @@ -668,4 +667,131 @@ mod tests { .unwrap(); assert_eq!(interval_value, json!("P1Y1M2DT0H0M1S")); } + + #[test] + fn test_generate_json_converter_schema() { + let mock_field = Field { + data_type: DataType::Boolean, + name: Default::default(), + sub_fields: Default::default(), + type_name: Default::default(), + }; + let fields = vec![ + Field { + data_type: DataType::Boolean, + name: "v1".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int16, + name: "v2".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int32, + name: "v3".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Float32, + name: "v4".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Decimal, + name: "v5".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Date, + name: "v6".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Varchar, + name: "v7".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Time, + name: "v8".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Interval, + name: "v9".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Struct(StructType::new(vec![ + ("a", DataType::Timestamp), + ("b", DataType::Timestamptz), + ( + "c", + DataType::Struct(StructType::new(vec![ + ("aa", DataType::Int64), + ("bb", DataType::Float64), + ])), + ), + ])), + name: "v10".into(), + sub_fields: vec![ + Field { + data_type: DataType::Timestamp, + name: "a".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Timestamptz, + name: "b".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Struct(StructType::new(vec![ + ("aa", DataType::Int64), + ("bb", DataType::Float64), + ])), + name: "c".into(), + sub_fields: vec![ + Field { + data_type: DataType::Int64, + name: "aa".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Float64, + name: "bb".into(), + ..mock_field.clone() + }, + ], + ..mock_field.clone() + }, + ], + ..mock_field.clone() + }, + Field { + data_type: DataType::List(Box::new(DataType::Bytea)), + name: "v11".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Jsonb, + name: "12".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Serial, + name: "13".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int256, + name: "14".into(), + ..mock_field.clone() + }, + ]; + let schema = generate_json_converter_schema(&fields, "test").to_string(); + let ans = r#"{"fields":[{"field":"v1","optional":"true","type":"boolean"},{"field":"v2","optional":"true","type":"int16"},{"field":"v3","optional":"true","type":"int32"},{"field":"v4","optional":"true","type":"float"},{"field":"v5","optional":"true","type":"string"},{"field":"v6","optional":"true","type":"int32"},{"field":"v7","optional":"true","type":"string"},{"field":"v8","optional":"true","type":"int64"},{"field":"v9","optional":"true","type":"string"},{"field":"v10","fields":[{"field":"a","optional":"true","type":"int64"},{"field":"b","optional":"true","type":"string"},{"field":"c","fields":[{"field":"aa","optional":"true","type":"int64"},{"field":"bb","optional":"true","type":"double"}],"optional":"true","type":"struct"}],"optional":"true","type":"struct"},{"field":"v11","items":{"type":"bytes"},"optional":"true","type":"array"},{"field":"12","optional":"true","type":"string"},{"field":"13","optional":"true","type":"int32"},{"field":"14","optional":"true","type":"string"}],"name":"test","optional":"false","type":"struct"}"#; + assert_eq!(schema, ans); + } } From e707686a5be95da1bd7d7abb510f0ee67507c7b8 Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 6 Sep 2023 08:14:01 +0000 Subject: [PATCH 06/21] fix key fields --- e2e_test/sink/kafka/upsert_schema1.result | 20 +++++++-------- e2e_test/sink/kafka/upsert_schema2.result | 22 ++++++++--------- e2e_test/sink/kafka/upsert_schema3.result | 24 +++++++++--------- src/connector/src/sink/utils.rs | 30 ++++++++++++++++++----- test | 12 +++++++++ 5 files changed, 69 insertions(+), 39 deletions(-) create mode 100644 test diff --git a/e2e_test/sink/kafka/upsert_schema1.result b/e2e_test/sink/kafka/upsert_schema1.result index 44effd0cc56d..2af049c82626 100644 --- a/e2e_test/sink/kafka/upsert_schema1.result +++ b/e2e_test/sink/kafka/upsert_schema1.result @@ -1,10 +1,10 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema2.result b/e2e_test/sink/kafka/upsert_schema2.result index f81299a9b99c..1231414b5f7b 100644 --- a/e2e_test/sink/kafka/upsert_schema2.result +++ b/e2e_test/sink/kafka/upsert_schema2.result @@ -1,11 +1,11 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema3.result b/e2e_test/sink/kafka/upsert_schema3.result index 90165cd8e79b..7a32af185f63 100644 --- a/e2e_test/sink/kafka/upsert_schema3.result +++ b/e2e_test/sink/kafka/upsert_schema3.result @@ -1,12 +1,12 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index 3580a55cffb5..b9965db8f68e 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -425,12 +425,15 @@ pub async fn gen_upsert_message_stream<'a>( _opts: UpsertAdapterOpts, ) { for (op, row) in chunk.rows() { - let event_key_object = gen_event_object( - schema, - Value::Object(pk_to_json(row, &schema.fields, pk_indices)?), - enable_schema, - &schema_name, - ); + let event_key_object_inner = Value::Object(pk_to_json(row, &schema.fields, pk_indices)?); + let event_key_object = if enable_schema { + Some(json!({ + "schema": generate_json_converter_schema_with_indices(&schema.fields, schema_name.as_ref().unwrap(), pk_indices), + "payload": event_key_object_inner, + })) + } else { + Some(event_key_object_inner) + }; let event_object = match op { Op::Insert => gen_event_object( @@ -547,6 +550,21 @@ fn generate_json_converter_schema(fields: &[Field], name: &str) -> Value { }) } +/// Generate schema for Kafka's `JsonConverter` when `schema.enable` is true according +/// to indices +fn generate_json_converter_schema_with_indices( + fields: &[Field], + name: &str, + indices: &[usize], +) -> Value { + json!({ + "type": "struct", + "fields": indices.iter().map(|i| json_converter_field_to_json(&fields[*i])).collect::>(), + "optional": "false", + "name": name, + }) +} + #[cfg(test)] mod tests { diff --git a/test b/test new file mode 100644 index 000000000000..7a32af185f63 --- /dev/null +++ b/test @@ -0,0 +1,12 @@ +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} From 3137eafc34bf9c1df0a44c9b19064a5347c831fb Mon Sep 17 00:00:00 2001 From: wu Date: Wed, 6 Sep 2023 08:16:31 +0000 Subject: [PATCH 07/21] fix check --- src/connector/src/sink/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index b9965db8f68e..bfd2d1c34101 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -805,7 +805,7 @@ mod tests { Field { data_type: DataType::Int256, name: "14".into(), - ..mock_field.clone() + ..mock_field }, ]; let schema = generate_json_converter_schema(&fields, "test").to_string(); From 19459cf74d1e27e2a55add2cb0aa9f6feb36fe84 Mon Sep 17 00:00:00 2001 From: wu Date: Fri, 15 Sep 2023 08:55:57 +0000 Subject: [PATCH 08/21] update with new design --- src/connector/src/sink/encoder/json.rs | 2 +- src/connector/src/sink/encoder/mod.rs | 2 +- src/connector/src/sink/utils.rs | 105 +++++++++++-------------- 3 files changed, 46 insertions(+), 63 deletions(-) diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index e4fe775f6306..5d9def14e16e 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -86,7 +86,7 @@ impl SerTo for Value { } } -fn datum_to_json_object( +pub fn datum_to_json_object( field: &Field, datum: DatumRef<'_>, timestamp_handling_mode: TimestampHandlingMode, diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs index 83f185935a44..c94650bc5d4b 100644 --- a/src/connector/src/sink/encoder/mod.rs +++ b/src/connector/src/sink/encoder/mod.rs @@ -19,7 +19,7 @@ use crate::sink::Result; mod json; -pub use json::JsonEncoder; +pub use json::{datum_to_json_object, JsonEncoder}; /// Encode a row of a relation into /// * an object in json diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index 66dc90c19697..b9106fce03e6 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -16,6 +16,7 @@ use futures_async_stream::try_stream; use risingwave_common::array::stream_chunk::Op; use risingwave_common::array::StreamChunk; use risingwave_common::catalog::{Field, Schema}; +use risingwave_common::types::DataType; use serde_json::{json, Map, Value}; use tracing::warn; @@ -253,22 +254,6 @@ pub fn chunk_to_json(chunk: StreamChunk, schema: &Schema) -> Result> #[derive(Debug, Clone, Default)] pub struct UpsertAdapterOpts {} -fn gen_event_object( - schema: &Schema, - object: Value, - enable_schema: bool, - name: &Option, -) -> Option { - if enable_schema { - Some(json!({ - "schema": generate_json_converter_schema(&schema.fields, name.as_ref().unwrap()), - "payload": object, - })) - } else { - Some(object) - } -} - #[try_stream(ok = (Option, Option), error = SinkError)] pub async fn gen_upsert_message_stream<'a>( chunk: StreamChunk, @@ -279,24 +264,22 @@ pub async fn gen_upsert_message_stream<'a>( val_encoder: JsonEncoder<'a>, ) { for (op, row) in chunk.rows() { - let event_key_object_inner = Value::Object(key_encoder.encode(row)?; + let event_key_object_inner = Value::Object(key_encoder.encode(row)?); let event_key_object = if enable_schema { - Some(json!({ - "schema": generate_json_converter_schema_with_indices(&schema.fields, schema_name.as_ref().unwrap(), pk_indices), - "payload": event_key_object_inner, - })) + json_converter_gen_event_object( + key_encoder.schema(), + event_key_object_inner, + enable_schema, + &schema_name, + ) } else { Some(event_key_object_inner) }; let event_object = match op { - Op::Insert => gen_event_object( - schema, - Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?), + Op::Insert => json_converter_gen_event_object( + val_encoder.schema(), + Value::Object(val_encoder.encode(row)?), enable_schema, &schema_name, ), @@ -305,13 +288,9 @@ pub async fn gen_upsert_message_stream<'a>( // upsert semantic does not require update delete event continue; } - Op::UpdateInsert => gen_event_object( - schema, - Value::Object(record_to_json( - row, - &schema.fields, - TimestampHandlingMode::Milli, - )?), + Op::UpdateInsert => json_converter_gen_event_object( + val_encoder.schema(), + Value::Object(val_encoder.encode(row)?), enable_schema, &schema_name, ), @@ -342,6 +321,27 @@ pub async fn gen_append_only_message_stream<'a>( } } +fn json_converter_gen_event_object( + schema: &Schema, + object: Value, + enable_schema: bool, + name: &Option, +) -> Option { + if enable_schema { + Some(json!({ + "schema": { + "type": "struct", + "fields": schema.fields().iter().map(json_converter_field_to_json).collect::>(), + "optional": "false", + "name": name, + }, + "payload": object, + })) + } else { + Some(object) + } +} + // reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSchema.java#L39 fn json_converter_field_to_json(field: &Field) -> Value { let mut mapping = Map::with_capacity(4); @@ -390,37 +390,13 @@ fn json_converter_field_to_json(field: &Field) -> Value { json!(mapping) } -/// Generate schema for Kafka's `JsonConverter` when `schema.enable` is true -fn generate_json_converter_schema(fields: &[Field], name: &str) -> Value { - json!({ - "type": "struct", - "fields": fields.iter().map(json_converter_field_to_json).collect::>(), - "optional": "false", - "name": name, - }) -} - -/// Generate schema for Kafka's `JsonConverter` when `schema.enable` is true according -/// to indices -fn generate_json_converter_schema_with_indices( - fields: &[Field], - name: &str, - indices: &[usize], -) -> Value { - json!({ - "type": "struct", - "fields": indices.iter().map(|i| json_converter_field_to_json(&fields[*i])).collect::>(), - "optional": "false", - "name": name, - }) -} - #[cfg(test)] mod tests { use risingwave_common::types::{DataType, Interval, ScalarImpl, StructType, Time, Timestamp}; use super::*; + use crate::sink::encoder::datum_to_json_object; #[test] fn test_to_json_basic_type() { let mock_field = Field { @@ -658,7 +634,14 @@ mod tests { ..mock_field }, ]; - let schema = generate_json_converter_schema(&fields, "test").to_string(); + + let schema = json!({ + "type": "struct", + "fields": fields.iter().map(json_converter_field_to_json).collect::>(), + "optional": "false", + "name": "test", + }) + .to_string(); let ans = r#"{"fields":[{"field":"v1","optional":"true","type":"boolean"},{"field":"v2","optional":"true","type":"int16"},{"field":"v3","optional":"true","type":"int32"},{"field":"v4","optional":"true","type":"float"},{"field":"v5","optional":"true","type":"string"},{"field":"v6","optional":"true","type":"int32"},{"field":"v7","optional":"true","type":"string"},{"field":"v8","optional":"true","type":"int64"},{"field":"v9","optional":"true","type":"string"},{"field":"v10","fields":[{"field":"a","optional":"true","type":"int64"},{"field":"b","optional":"true","type":"string"},{"field":"c","fields":[{"field":"aa","optional":"true","type":"int64"},{"field":"bb","optional":"true","type":"double"}],"optional":"true","type":"struct"}],"optional":"true","type":"struct"},{"field":"v11","items":{"type":"bytes"},"optional":"true","type":"array"},{"field":"12","optional":"true","type":"string"},{"field":"13","optional":"true","type":"int32"},{"field":"14","optional":"true","type":"string"}],"name":"test","optional":"false","type":"struct"}"#; assert_eq!(schema, ans); } From f8445855f102ed7a78f4c4a411c5b4ae38e92a92 Mon Sep 17 00:00:00 2001 From: wu Date: Fri, 15 Sep 2023 09:32:08 +0000 Subject: [PATCH 09/21] fix key schema --- src/connector/src/sink/utils.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index b9106fce03e6..00a22c08324b 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -267,7 +267,7 @@ pub async fn gen_upsert_message_stream<'a>( let event_key_object_inner = Value::Object(key_encoder.encode(row)?); let event_key_object = if enable_schema { json_converter_gen_event_object( - key_encoder.schema(), + &key_encoder, event_key_object_inner, enable_schema, &schema_name, @@ -278,7 +278,7 @@ pub async fn gen_upsert_message_stream<'a>( let event_object = match op { Op::Insert => json_converter_gen_event_object( - val_encoder.schema(), + &val_encoder, Value::Object(val_encoder.encode(row)?), enable_schema, &schema_name, @@ -289,7 +289,7 @@ pub async fn gen_upsert_message_stream<'a>( continue; } Op::UpdateInsert => json_converter_gen_event_object( - val_encoder.schema(), + &key_encoder, Value::Object(val_encoder.encode(row)?), enable_schema, &schema_name, @@ -322,16 +322,20 @@ pub async fn gen_append_only_message_stream<'a>( } fn json_converter_gen_event_object( - schema: &Schema, + encoder: &JsonEncoder<'_>, object: Value, enable_schema: bool, name: &Option, ) -> Option { + let fields = encoder.schema().fields(); if enable_schema { Some(json!({ "schema": { "type": "struct", - "fields": schema.fields().iter().map(json_converter_field_to_json).collect::>(), + "fields": match encoder.col_indices() { + Some(indices) => indices.iter().map(|i| json_converter_field_to_json(&fields[*i])).collect::>(), + _ => fields.iter().map(json_converter_field_to_json).collect::>(), + }, "optional": "false", "name": name, }, From 7ec7af9ca3b2e610d0bb10274991f45de3b443af Mon Sep 17 00:00:00 2001 From: wu Date: Fri, 15 Sep 2023 09:46:43 +0000 Subject: [PATCH 10/21] fix encoder --- src/connector/src/sink/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/src/sink/utils.rs b/src/connector/src/sink/utils.rs index 00a22c08324b..16d8b3ca8c7d 100644 --- a/src/connector/src/sink/utils.rs +++ b/src/connector/src/sink/utils.rs @@ -289,7 +289,7 @@ pub async fn gen_upsert_message_stream<'a>( continue; } Op::UpdateInsert => json_converter_gen_event_object( - &key_encoder, + &val_encoder, Value::Object(val_encoder.encode(row)?), enable_schema, &schema_name, From 8f1cc904102fe6ca4178330b3fb5522d78f37422 Mon Sep 17 00:00:00 2001 From: wu Date: Fri, 15 Sep 2023 18:10:22 +0800 Subject: [PATCH 11/21] Delete useless file --- test | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 test diff --git a/test b/test deleted file mode 100644 index 7a32af185f63..000000000000 --- a/test +++ /dev/null @@ -1,12 +0,0 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} From 2ddffdc7f877a483be68c42b70d334095cbeb47a Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:01:48 +0800 Subject: [PATCH 12/21] refactor kafka upsert json schema --- src/connector/src/sink/encoder/json.rs | 109 ++++++++++++++++++++++-- src/connector/src/sink/encoder/mod.rs | 1 + src/connector/src/sink/formatter/mod.rs | 31 +++++-- src/connector/src/sink/kafka.rs | 9 +- 4 files changed, 129 insertions(+), 21 deletions(-) diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index 629a69482e90..d34f7c49841a 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -18,6 +18,7 @@ use base64::engine::general_purpose; use base64::Engine as _; use chrono::{Datelike, Timelike}; use indexmap::IndexMap; +use itertools::Itertools; use risingwave_common::array::{ArrayError, ArrayResult}; use risingwave_common::catalog::{Field, Schema}; use risingwave_common::row::Row; @@ -62,6 +63,20 @@ impl JsonEncoder { custom_json_type: CustomJsonType::Doris(map), } } + + pub fn new_with_kafka( + schema: Schema, + col_indices: Option>, + timestamp_handling_mode: TimestampHandlingMode, + schema_name: String, + ) -> Self { + Self { + schema, + col_indices, + timestamp_handling_mode, + custom_json_type: CustomJsonType::Kafka(schema_name), + } + } } impl RowEncoder for JsonEncoder { @@ -81,19 +96,31 @@ impl RowEncoder for JsonEncoder { col_indices: impl Iterator, ) -> Result { let mut mappings = Map::with_capacity(self.schema.len()); - for idx in col_indices { - let field = &self.schema[idx]; + let col_indices = col_indices.collect_vec(); + for idx in &col_indices { + let field = &self.schema[*idx]; let key = field.name.clone(); let value = datum_to_json_object( field, - row.datum_at(idx), + row.datum_at(*idx), self.timestamp_handling_mode, &self.custom_json_type, ) .map_err(|e| SinkError::Encode(e.to_string()))?; mappings.insert(key, value); } - Ok(mappings) + + Ok(match &self.custom_json_type { + CustomJsonType::Kafka(name) => json_converter_with_schema( + Value::Object(mappings), + Some(name.to_owned()), + col_indices + .into_iter() + .map(|i| &self.schema()[i]) + .collect_vec(), + ), + _ => mappings, + }) } } @@ -162,7 +189,7 @@ pub fn datum_to_json_object( } json!(v_string) } - CustomJsonType::None => { + _ => { json!(v.to_text()) } }, @@ -178,11 +205,12 @@ pub fn datum_to_json_object( json!(v.0.num_seconds_from_midnight() as i64 * 1000) } (DataType::Date, ScalarRefImpl::Date(v)) => match custom_json_type { - CustomJsonType::None => json!(v.0.num_days_from_ce()), CustomJsonType::Doris(_) => { let a = v.0.format("%Y-%m-%d").to_string(); json!(a) } + + _ => json!(v.0.num_days_from_ce()), }, (DataType::Timestamp, ScalarRefImpl::Timestamp(v)) => match timestamp_handling_mode { TimestampHandlingMode::Milli => json!(v.0.timestamp_millis()), @@ -234,7 +262,7 @@ pub fn datum_to_json_object( ArrayError::internal(format!("Json to string err{:?}", err)) })?) } - CustomJsonType::None => { + _ => { let mut map = Map::with_capacity(st.len()); for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( st.iter() @@ -262,6 +290,73 @@ pub fn datum_to_json_object( Ok(value) } +fn json_converter_with_schema( + object: Value, + name: Option, + fields: Vec<&Field>, +) -> Map { + let mut mapping = Map::with_capacity(2); + mapping.insert( + "schema".to_string(), + json!({ + "type": "struct", + "fields": fields.into_iter().map(json_converter_field_to_json).collect::>(), + "optional": "false", + "name": name, + }), + ); + mapping.insert("payload".to_string(), object); + mapping +} + +// reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSchema.java#L39 +fn json_converter_field_to_json(field: &Field) -> Value { + let mut mapping = Map::with_capacity(4); + let type_mapping = |rw_type: &DataType| match rw_type { + DataType::Boolean => "boolean", + DataType::Int16 => "int16", + DataType::Int32 => "int32", + DataType::Int64 => "int64", + DataType::Float32 => "float", + DataType::Float64 => "double", + DataType::Decimal => "string", + DataType::Date => "int32", + DataType::Varchar => "string", + DataType::Time => "int64", + DataType::Timestamp => "int64", + DataType::Timestamptz => "string", + DataType::Interval => "string", + DataType::Struct(_) => "struct", + DataType::List(_) => "array", + DataType::Bytea => "bytes", + DataType::Jsonb => "string", + DataType::Serial => "int32", + DataType::Int256 => "string", + }; + mapping.insert("type".into(), json!(type_mapping(&field.data_type))); + mapping.insert("optional".into(), json!("true")); + mapping.insert("field".into(), json!(field.name)); + match &field.data_type { + DataType::Struct(_) => { + let mut sub_fields = Vec::new(); + for sub_field in &field.sub_fields { + sub_fields.push(json_converter_field_to_json(sub_field)); + } + mapping.insert("fields".into(), json!(sub_fields)); + } + DataType::List(list_type) => { + mapping.insert( + "items".into(), + json!({ + "type": type_mapping(list_type), + }), + ); + } + _ => {} + } + json!(mapping) +} + #[cfg(test)] mod tests { diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs index b0a13606948d..f2fce50eeb8a 100644 --- a/src/connector/src/sink/encoder/mod.rs +++ b/src/connector/src/sink/encoder/mod.rs @@ -90,6 +90,7 @@ pub enum TimestampHandlingMode { #[derive(Clone)] pub enum CustomJsonType { Doris(HashMap), + Kafka(String), None, } diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 61fe7a830c0a..55db99048f0f 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -106,12 +106,31 @@ impl SinkFormatterImpl { ))) } SinkFormat::Upsert => { - let key_encoder = JsonEncoder::new( - schema.clone(), - Some(pk_indices), - TimestampHandlingMode::Milli, - ); - let val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); + let schema_enable = matches!(format_desc.options.get("schema.enable"), Some(s) if s.to_lowercase() == "true"); + let key_encoder = if schema_enable { + JsonEncoder::new_with_kafka( + schema.clone(), + Some(pk_indices), + TimestampHandlingMode::Milli, + format!("{}.{}", db_name, sink_from_name), + ) + } else { + JsonEncoder::new( + schema.clone(), + Some(pk_indices), + TimestampHandlingMode::Milli, + ) + }; + let val_encoder = if schema_enable { + JsonEncoder::new_with_kafka( + schema, + None, + TimestampHandlingMode::Milli, + format!("{}.{}", db_name, sink_from_name), + ) + } else { + JsonEncoder::new(schema, None, TimestampHandlingMode::Milli) + }; // Initialize the upsert_stream let formatter = UpsertFormatter::new(key_encoder, val_encoder); diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs index 665dda005b96..ef4efe88e43f 100644 --- a/src/connector/src/sink/kafka.rs +++ b/src/connector/src/sink/kafka.rs @@ -45,7 +45,7 @@ use crate::sink::{DummySinkCommitCoordinator, LogSinker, Result, SinkWriterParam use crate::source::kafka::{KafkaProperties, KafkaSplitEnumerator, PrivateLinkProducerContext}; use crate::source::{SourceEnumeratorContext, SplitEnumerator}; use crate::{ - deserialize_duration_from_string, deserialize_u32_from_string, dispatch_sink_formatter_impl, deserialize_bool_from_string + deserialize_duration_from_string, deserialize_u32_from_string, dispatch_sink_formatter_impl, }; pub const KAFKA_SINK: &str = "kafka"; @@ -228,13 +228,6 @@ pub struct KafkaConfig { )] pub retry_interval: Duration, - #[serde( - rename = "schema.enable", - default, - deserialize_with = "deserialize_bool_from_string" - )] - pub schema_enable: bool, - /// We have parsed the primary key for an upsert kafka sink into a `usize` vector representing /// the indices of the pk columns in the frontend, so we simply store the primary key here /// as a string. From 280bfbab6a671694db62aa99d503bb80cca95ac9 Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Tue, 17 Oct 2023 15:13:47 +0800 Subject: [PATCH 13/21] fix test --- e2e_test/sink/kafka/create_sink.slt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e_test/sink/kafka/create_sink.slt b/e2e_test/sink/kafka/create_sink.slt index f293ad586354..89f04282f6f1 100644 --- a/e2e_test/sink/kafka/create_sink.slt +++ b/e2e_test/sink/kafka/create_sink.slt @@ -104,7 +104,8 @@ create sink si_kafka_upsert_schema from t_kafka with ( topic = 'test-rw-sink-upsert-schema', type = 'upsert', primary_key = 'id', - schema.enable = true, +) format upsert encode json ( + schema.enable = true ); statement ok From 4f21363ef3d013bbd74476c547772f8585f584ed Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Tue, 17 Oct 2023 17:22:22 +0800 Subject: [PATCH 14/21] compact code --- src/connector/src/sink/encoder/json.rs | 80 +++++-------------------- src/connector/src/sink/encoder/mod.rs | 6 +- src/connector/src/sink/formatter/mod.rs | 43 ++++++------- 3 files changed, 37 insertions(+), 92 deletions(-) diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index d34f7c49841a..0865b002cd7b 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -26,7 +26,8 @@ use risingwave_common::types::{DataType, DatumRef, Decimal, ScalarRefImpl, ToTex use risingwave_common::util::iter_util::ZipEqDebug; use serde_json::{json, Map, Value}; -use super::{CustomJsonType, Result, RowEncoder, SerTo, TimestampHandlingMode}; +use super::{CustomJsonType, KafkaConnectParams, Result, RowEncoder, SerTo, TimestampHandlingMode}; +use crate::sink::formatter::debezium_json::field_to_json; use crate::sink::SinkError; pub struct JsonEncoder { @@ -34,6 +35,7 @@ pub struct JsonEncoder { col_indices: Option>, timestamp_handling_mode: TimestampHandlingMode, custom_json_type: CustomJsonType, + kafka_connect: Option, } impl JsonEncoder { @@ -47,6 +49,7 @@ impl JsonEncoder { col_indices, timestamp_handling_mode, custom_json_type: CustomJsonType::None, + kafka_connect: None, } } @@ -61,20 +64,14 @@ impl JsonEncoder { col_indices, timestamp_handling_mode, custom_json_type: CustomJsonType::Doris(map), + kafka_connect: None, } } - pub fn new_with_kafka( - schema: Schema, - col_indices: Option>, - timestamp_handling_mode: TimestampHandlingMode, - schema_name: String, - ) -> Self { + pub fn with_kafka_connect(self, kafka_connect: KafkaConnectParams) -> Self { Self { - schema, - col_indices, - timestamp_handling_mode, - custom_json_type: CustomJsonType::Kafka(schema_name), + kafka_connect: Some(kafka_connect), + ..self } } } @@ -110,16 +107,17 @@ impl RowEncoder for JsonEncoder { mappings.insert(key, value); } - Ok(match &self.custom_json_type { - CustomJsonType::Kafka(name) => json_converter_with_schema( + Ok(if let Some(param) = &self.kafka_connect { + json_converter_with_schema( Value::Object(mappings), - Some(name.to_owned()), + Some(param.schema_name.to_owned()), col_indices .into_iter() .map(|i| &self.schema()[i]) .collect_vec(), - ), - _ => mappings, + ) + } else { + mappings }) } } @@ -300,7 +298,7 @@ fn json_converter_with_schema( "schema".to_string(), json!({ "type": "struct", - "fields": fields.into_iter().map(json_converter_field_to_json).collect::>(), + "fields": fields.into_iter().map(field_to_json).collect_vec(), "optional": "false", "name": name, }), @@ -309,54 +307,6 @@ fn json_converter_with_schema( mapping } -// reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSchema.java#L39 -fn json_converter_field_to_json(field: &Field) -> Value { - let mut mapping = Map::with_capacity(4); - let type_mapping = |rw_type: &DataType| match rw_type { - DataType::Boolean => "boolean", - DataType::Int16 => "int16", - DataType::Int32 => "int32", - DataType::Int64 => "int64", - DataType::Float32 => "float", - DataType::Float64 => "double", - DataType::Decimal => "string", - DataType::Date => "int32", - DataType::Varchar => "string", - DataType::Time => "int64", - DataType::Timestamp => "int64", - DataType::Timestamptz => "string", - DataType::Interval => "string", - DataType::Struct(_) => "struct", - DataType::List(_) => "array", - DataType::Bytea => "bytes", - DataType::Jsonb => "string", - DataType::Serial => "int32", - DataType::Int256 => "string", - }; - mapping.insert("type".into(), json!(type_mapping(&field.data_type))); - mapping.insert("optional".into(), json!("true")); - mapping.insert("field".into(), json!(field.name)); - match &field.data_type { - DataType::Struct(_) => { - let mut sub_fields = Vec::new(); - for sub_field in &field.sub_fields { - sub_fields.push(json_converter_field_to_json(sub_field)); - } - mapping.insert("fields".into(), json!(sub_fields)); - } - DataType::List(list_type) => { - mapping.insert( - "items".into(), - json!({ - "type": type_mapping(list_type), - }), - ); - } - _ => {} - } - json!(mapping) -} - #[cfg(test)] mod tests { diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs index f2fce50eeb8a..389359a2284e 100644 --- a/src/connector/src/sink/encoder/mod.rs +++ b/src/connector/src/sink/encoder/mod.rs @@ -90,7 +90,6 @@ pub enum TimestampHandlingMode { #[derive(Clone)] pub enum CustomJsonType { Doris(HashMap), - Kafka(String), None, } @@ -132,3 +131,8 @@ impl From for super::SinkError { Self::Encode(value.to_string()) } } + +#[derive(Clone)] +pub struct KafkaConnectParams { + pub schema_name: String, +} diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 55db99048f0f..5b754bd7cb6d 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -18,7 +18,7 @@ use risingwave_common::array::StreamChunk; use crate::sink::{Result, SinkError}; mod append_only; -mod debezium_json; +pub mod debezium_json; mod upsert; pub use append_only::AppendOnlyFormatter; @@ -27,6 +27,7 @@ use risingwave_common::catalog::Schema; pub use upsert::UpsertFormatter; use super::catalog::{SinkEncode, SinkFormat, SinkFormatDesc}; +use super::encoder::KafkaConnectParams; use crate::sink::encoder::{JsonEncoder, TimestampHandlingMode}; /// Transforms a `StreamChunk` into a sequence of key-value pairs according a specific format, @@ -106,31 +107,21 @@ impl SinkFormatterImpl { ))) } SinkFormat::Upsert => { - let schema_enable = matches!(format_desc.options.get("schema.enable"), Some(s) if s.to_lowercase() == "true"); - let key_encoder = if schema_enable { - JsonEncoder::new_with_kafka( - schema.clone(), - Some(pk_indices), - TimestampHandlingMode::Milli, - format!("{}.{}", db_name, sink_from_name), - ) - } else { - JsonEncoder::new( - schema.clone(), - Some(pk_indices), - TimestampHandlingMode::Milli, - ) - }; - let val_encoder = if schema_enable { - JsonEncoder::new_with_kafka( - schema, - None, - TimestampHandlingMode::Milli, - format!("{}.{}", db_name, sink_from_name), - ) - } else { - JsonEncoder::new(schema, None, TimestampHandlingMode::Milli) - }; + let mut key_encoder = JsonEncoder::new( + schema.clone(), + Some(pk_indices), + TimestampHandlingMode::Milli, + ); + let mut val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); + + if matches!(format_desc.options.get("schema.enable"), Some(s) if s.to_lowercase() == "true") + { + let kafka_connect = KafkaConnectParams { + schema_name: format!("{}.{}", db_name, sink_from_name), + }; + key_encoder = key_encoder.with_kafka_connect(kafka_connect.clone()); + val_encoder = val_encoder.with_kafka_connect(kafka_connect); + } // Initialize the upsert_stream let formatter = UpsertFormatter::new(key_encoder, val_encoder); From 4675de98812848ab3d3e2d5cafdad3c4fbe67856 Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Tue, 17 Oct 2023 18:21:53 +0800 Subject: [PATCH 15/21] refactor code --- e2e_test/sink/kafka/create_sink.slt | 3 +- src/connector/src/sink/encoder/json.rs | 38 +++++++++++-------------- src/connector/src/sink/formatter/mod.rs | 26 +++++++++++------ 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/e2e_test/sink/kafka/create_sink.slt b/e2e_test/sink/kafka/create_sink.slt index 89f04282f6f1..25e3a59fdff3 100644 --- a/e2e_test/sink/kafka/create_sink.slt +++ b/e2e_test/sink/kafka/create_sink.slt @@ -102,10 +102,9 @@ create sink si_kafka_upsert_schema from t_kafka with ( connector = 'kafka', properties.bootstrap.server = '127.0.0.1:29092', topic = 'test-rw-sink-upsert-schema', - type = 'upsert', primary_key = 'id', ) format upsert encode json ( - schema.enable = true + schemas.enable = true ); statement ok diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index 0865b002cd7b..115568915d63 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -111,10 +111,7 @@ impl RowEncoder for JsonEncoder { json_converter_with_schema( Value::Object(mappings), Some(param.schema_name.to_owned()), - col_indices - .into_iter() - .map(|i| &self.schema()[i]) - .collect_vec(), + col_indices.into_iter().map(|i| &self.schema[i]), ) } else { mappings @@ -172,6 +169,9 @@ pub fn datum_to_json_object( json!(v) } (DataType::Decimal, ScalarRefImpl::Decimal(mut v)) => match custom_json_type { + CustomJsonType::None => { + json!(v.to_text()) + } CustomJsonType::Doris(map) => { if !matches!(v, Decimal::Normalized(_)) { return Err(ArrayError::internal( @@ -187,9 +187,6 @@ pub fn datum_to_json_object( } json!(v_string) } - _ => { - json!(v.to_text()) - } }, (DataType::Timestamptz, ScalarRefImpl::Timestamptz(v)) => { // risingwave's timestamp with timezone is stored in UTC and does not maintain the @@ -203,12 +200,11 @@ pub fn datum_to_json_object( json!(v.0.num_seconds_from_midnight() as i64 * 1000) } (DataType::Date, ScalarRefImpl::Date(v)) => match custom_json_type { + CustomJsonType::None => json!(v.0.num_days_from_ce()), CustomJsonType::Doris(_) => { let a = v.0.format("%Y-%m-%d").to_string(); json!(a) } - - _ => json!(v.0.num_days_from_ce()), }, (DataType::Timestamp, ScalarRefImpl::Timestamp(v)) => match timestamp_handling_mode { TimestampHandlingMode::Milli => json!(v.0.timestamp_millis()), @@ -241,9 +237,8 @@ pub fn datum_to_json_object( } (DataType::Struct(st), ScalarRefImpl::Struct(struct_ref)) => { match custom_json_type { - CustomJsonType::Doris(_) => { - // We need to ensure that the order of elements in the json matches the insertion order. - let mut map = IndexMap::with_capacity(st.len()); + CustomJsonType::None => { + let mut map = Map::with_capacity(st.len()); for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( st.iter() .map(|(name, dt)| Field::with_name(dt.clone(), name)), @@ -256,12 +251,11 @@ pub fn datum_to_json_object( )?; map.insert(sub_field.name.clone(), value); } - Value::String(serde_json::to_string(&map).map_err(|err| { - ArrayError::internal(format!("Json to string err{:?}", err)) - })?) + json!(map) } - _ => { - let mut map = Map::with_capacity(st.len()); + CustomJsonType::Doris(_) => { + // We need to ensure that the order of elements in the json matches the insertion order. + let mut map = IndexMap::with_capacity(st.len()); for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( st.iter() .map(|(name, dt)| Field::with_name(dt.clone(), name)), @@ -274,7 +268,9 @@ pub fn datum_to_json_object( )?; map.insert(sub_field.name.clone(), value); } - json!(map) + Value::String(serde_json::to_string(&map).map_err(|err| { + ArrayError::internal(format!("Json to string err{:?}", err)) + })?) } } } @@ -288,17 +284,17 @@ pub fn datum_to_json_object( Ok(value) } -fn json_converter_with_schema( +fn json_converter_with_schema<'a>( object: Value, name: Option, - fields: Vec<&Field>, + fields: impl Iterator, ) -> Map { let mut mapping = Map::with_capacity(2); mapping.insert( "schema".to_string(), json!({ "type": "struct", - "fields": fields.into_iter().map(field_to_json).collect_vec(), + "fields": fields.map(field_to_json).collect_vec(), "optional": "false", "name": name, }), diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 5b754bd7cb6d..64f44c9a4cc9 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -114,14 +114,24 @@ impl SinkFormatterImpl { ); let mut val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); - if matches!(format_desc.options.get("schema.enable"), Some(s) if s.to_lowercase() == "true") - { - let kafka_connect = KafkaConnectParams { - schema_name: format!("{}.{}", db_name, sink_from_name), - }; - key_encoder = key_encoder.with_kafka_connect(kafka_connect.clone()); - val_encoder = val_encoder.with_kafka_connect(kafka_connect); - } + if let Some(s) = format_desc.options.get("schemas.enable") { + match s.to_lowercase().as_str() { + "true" => { + let kafka_connect = KafkaConnectParams { + schema_name: format!("{}.{}", db_name, sink_from_name), + }; + key_encoder = key_encoder.with_kafka_connect(kafka_connect.clone()); + val_encoder = val_encoder.with_kafka_connect(kafka_connect); + } + "false" => (), + other => { + return Err(SinkError::Config(anyhow!( + "schemas.enable is expected to be `true` or `false`, got {}", + other + ))) + } + } + }; // Initialize the upsert_stream let formatter = UpsertFormatter::new(key_encoder, val_encoder); From ecbafe34dadb3e4c1adf0d63c64a85dabeaa454f Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Tue, 17 Oct 2023 18:27:09 +0800 Subject: [PATCH 16/21] refactor code --- src/connector/src/sink/encoder/json.rs | 24 ++++++++++++------------ src/connector/src/sink/formatter/mod.rs | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index 115568915d63..85727dcc97da 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -169,9 +169,6 @@ pub fn datum_to_json_object( json!(v) } (DataType::Decimal, ScalarRefImpl::Decimal(mut v)) => match custom_json_type { - CustomJsonType::None => { - json!(v.to_text()) - } CustomJsonType::Doris(map) => { if !matches!(v, Decimal::Normalized(_)) { return Err(ArrayError::internal( @@ -187,6 +184,9 @@ pub fn datum_to_json_object( } json!(v_string) } + CustomJsonType::None => { + json!(v.to_text()) + } }, (DataType::Timestamptz, ScalarRefImpl::Timestamptz(v)) => { // risingwave's timestamp with timezone is stored in UTC and does not maintain the @@ -237,8 +237,9 @@ pub fn datum_to_json_object( } (DataType::Struct(st), ScalarRefImpl::Struct(struct_ref)) => { match custom_json_type { - CustomJsonType::None => { - let mut map = Map::with_capacity(st.len()); + CustomJsonType::Doris(_) => { + // We need to ensure that the order of elements in the json matches the insertion order. + let mut map = IndexMap::with_capacity(st.len()); for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( st.iter() .map(|(name, dt)| Field::with_name(dt.clone(), name)), @@ -251,11 +252,12 @@ pub fn datum_to_json_object( )?; map.insert(sub_field.name.clone(), value); } - json!(map) + Value::String(serde_json::to_string(&map).map_err(|err| { + ArrayError::internal(format!("Json to string err{:?}", err)) + })?) } - CustomJsonType::Doris(_) => { - // We need to ensure that the order of elements in the json matches the insertion order. - let mut map = IndexMap::with_capacity(st.len()); + CustomJsonType::None => { + let mut map = Map::with_capacity(st.len()); for (sub_datum_ref, sub_field) in struct_ref.iter_fields_ref().zip_eq_debug( st.iter() .map(|(name, dt)| Field::with_name(dt.clone(), name)), @@ -268,9 +270,7 @@ pub fn datum_to_json_object( )?; map.insert(sub_field.name.clone(), value); } - Value::String(serde_json::to_string(&map).map_err(|err| { - ArrayError::internal(format!("Json to string err{:?}", err)) - })?) + json!(map) } } } diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 64f44c9a4cc9..1865461a65e5 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -123,7 +123,7 @@ impl SinkFormatterImpl { key_encoder = key_encoder.with_kafka_connect(kafka_connect.clone()); val_encoder = val_encoder.with_kafka_connect(kafka_connect); } - "false" => (), + "false" => {} other => { return Err(SinkError::Config(anyhow!( "schemas.enable is expected to be `true` or `false`, got {}", From dc63280454e31a620ad7bfe6fa93e7a1a13e7369 Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Wed, 18 Oct 2023 10:27:03 +0800 Subject: [PATCH 17/21] remove unnecessary Option --- src/connector/src/sink/encoder/json.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index 85727dcc97da..fecb66fee767 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -110,7 +110,7 @@ impl RowEncoder for JsonEncoder { Ok(if let Some(param) = &self.kafka_connect { json_converter_with_schema( Value::Object(mappings), - Some(param.schema_name.to_owned()), + param.schema_name.to_owned(), col_indices.into_iter().map(|i| &self.schema[i]), ) } else { @@ -286,7 +286,7 @@ pub fn datum_to_json_object( fn json_converter_with_schema<'a>( object: Value, - name: Option, + name: String, fields: impl Iterator, ) -> Map { let mut mapping = Map::with_capacity(2); From 8675aae864e0e14332372195013b56a838deb71e Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:19:33 +0800 Subject: [PATCH 18/21] fix kafka json schema --- src/common/src/types/struct_type.rs | 5 ++ src/connector/src/sink/encoder/json.rs | 69 +++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/src/common/src/types/struct_type.rs b/src/common/src/types/struct_type.rs index 239f506db826..c986c9a16299 100644 --- a/src/common/src/types/struct_type.rs +++ b/src/common/src/types/struct_type.rs @@ -94,6 +94,11 @@ impl StructType { self.0.field_types.is_empty() } + /// Returns `true` if the struct field is unnamed. + pub fn is_unnamed(&self) -> bool { + self.0.field_names.is_empty() + } + /// Gets an iterator over the names of the fields. /// /// If the struct field is unnamed, the iterator returns **no names**. diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index fecb66fee767..ca2faa2c40ec 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -27,7 +27,6 @@ use risingwave_common::util::iter_util::ZipEqDebug; use serde_json::{json, Map, Value}; use super::{CustomJsonType, KafkaConnectParams, Result, RowEncoder, SerTo, TimestampHandlingMode}; -use crate::sink::formatter::debezium_json::field_to_json; use crate::sink::SinkError; pub struct JsonEncoder { @@ -294,7 +293,7 @@ fn json_converter_with_schema<'a>( "schema".to_string(), json!({ "type": "struct", - "fields": fields.map(field_to_json).collect_vec(), + "fields": fields.map(field_as_json_schema).collect_vec(), "optional": "false", "name": name, }), @@ -303,6 +302,72 @@ fn json_converter_with_schema<'a>( mapping } +// reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonSchema.java#L39 +pub(crate) fn schema_type_mapping(rw_type: &DataType) -> &'static str { + match rw_type { + DataType::Boolean => "boolean", + DataType::Int16 => "int16", + DataType::Int32 => "int32", + DataType::Int64 => "int64", + DataType::Float32 => "float", + DataType::Float64 => "double", + DataType::Decimal => "string", + DataType::Date => "int32", + DataType::Varchar => "string", + DataType::Time => "int64", + DataType::Timestamp => "int64", + DataType::Timestamptz => "string", + DataType::Interval => "string", + DataType::Struct(_) => "struct", + DataType::List(_) => "array", + DataType::Bytea => "bytes", + DataType::Jsonb => "string", + DataType::Serial => "int32", + DataType::Int256 => "string", + } +} + +fn type_as_json_schema(rw_type: &DataType) -> Map { + let mut mapping = Map::with_capacity(4); // type + optional + fields/items + field + mapping.insert("type".to_string(), json!(schema_type_mapping(rw_type))); + mapping.insert("optional".to_string(), json!("true")); + match rw_type { + DataType::Struct(struct_type) => { + let mut sub_fields = Vec::new(); + let map_sub_struct = |(sub_name, sub_type): (String, &DataType)| { + let mut sub_mapping = type_as_json_schema(sub_type); + sub_mapping.insert("field".to_string(), json!(sub_name)); + sub_fields.push(sub_mapping); + }; + if struct_type.is_unnamed() { + // unnamed struct, rename with `f{idx+1}` + struct_type + .types() + .enumerate() + .map(|(idx, sub_type)| (format!("f{}", idx + 1), sub_type)) + .for_each(map_sub_struct); + } else { + struct_type + .iter() + .map(|(sub_name, sub_type)| (sub_name.to_string(), sub_type)) + .for_each(map_sub_struct); + } + mapping.insert("fields".to_string(), json!(sub_fields)); + } + DataType::List(sub_type) => { + mapping.insert("items".to_string(), json!(type_as_json_schema(sub_type))); + } + _ => {} + } + + mapping +} + +// reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java#L356 +fn field_as_json_schema(field: &Field) -> Value { + Value::Object(type_as_json_schema(&field.data_type)) +} + #[cfg(test)] mod tests { From 9d1fcde8d06f0993bed9d71040c41b04f91dd215 Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Wed, 18 Oct 2023 17:41:23 +0800 Subject: [PATCH 19/21] fix fields in schema root --- src/connector/src/sink/encoder/json.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index ca2faa2c40ec..df8f37251ba8 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -293,7 +293,11 @@ fn json_converter_with_schema<'a>( "schema".to_string(), json!({ "type": "struct", - "fields": fields.map(field_as_json_schema).collect_vec(), + "fields": fields.map(|field| { + let mut mapping = type_as_json_schema(&field.data_type); + mapping.insert("field".to_string(), json!(field.name)); + mapping + }).collect_vec(), "optional": "false", "name": name, }), From ffb0765629c5e404837e0df2d17d7a384fbac1cb Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Wed, 18 Oct 2023 19:22:02 +0800 Subject: [PATCH 20/21] refactor code --- src/common/src/types/struct_type.rs | 5 - src/connector/src/sink/encoder/json.rs | 158 +++++++++++++++++++++--- src/connector/src/sink/formatter/mod.rs | 12 +- 3 files changed, 145 insertions(+), 30 deletions(-) diff --git a/src/common/src/types/struct_type.rs b/src/common/src/types/struct_type.rs index c986c9a16299..239f506db826 100644 --- a/src/common/src/types/struct_type.rs +++ b/src/common/src/types/struct_type.rs @@ -94,11 +94,6 @@ impl StructType { self.0.field_types.is_empty() } - /// Returns `true` if the struct field is unnamed. - pub fn is_unnamed(&self) -> bool { - self.0.field_names.is_empty() - } - /// Gets an iterator over the names of the fields. /// /// If the struct field is unnamed, the iterator returns **no names**. diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index df8f37251ba8..636cb6f4199d 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -337,25 +337,14 @@ fn type_as_json_schema(rw_type: &DataType) -> Map { mapping.insert("optional".to_string(), json!("true")); match rw_type { DataType::Struct(struct_type) => { - let mut sub_fields = Vec::new(); - let map_sub_struct = |(sub_name, sub_type): (String, &DataType)| { - let mut sub_mapping = type_as_json_schema(sub_type); - sub_mapping.insert("field".to_string(), json!(sub_name)); - sub_fields.push(sub_mapping); - }; - if struct_type.is_unnamed() { - // unnamed struct, rename with `f{idx+1}` - struct_type - .types() - .enumerate() - .map(|(idx, sub_type)| (format!("f{}", idx + 1), sub_type)) - .for_each(map_sub_struct); - } else { - struct_type - .iter() - .map(|(sub_name, sub_type)| (sub_name.to_string(), sub_type)) - .for_each(map_sub_struct); - } + let sub_fields = struct_type + .iter() + .map(|(sub_name, sub_type)| { + let mut sub_mapping = type_as_json_schema(sub_type); + sub_mapping.insert("field".to_string(), json!(sub_name)); + sub_mapping + }) + .collect_vec(); mapping.insert("fields".to_string(), json!(sub_fields)); } DataType::List(sub_type) => { @@ -553,4 +542,135 @@ mod tests { .unwrap(); assert_eq!(interval_value, json!("{\"v3\":3,\"v2\":2,\"v1\":1}")); } + + #[test] + fn test_generate_json_converter_schema() { + let mock_field = Field { + data_type: DataType::Boolean, + name: Default::default(), + sub_fields: Default::default(), + type_name: Default::default(), + }; + let fields = vec![ + Field { + data_type: DataType::Boolean, + name: "v1".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int16, + name: "v2".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int32, + name: "v3".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Float32, + name: "v4".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Decimal, + name: "v5".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Date, + name: "v6".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Varchar, + name: "v7".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Time, + name: "v8".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Interval, + name: "v9".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Struct(StructType::new(vec![ + ("a", DataType::Timestamp), + ("b", DataType::Timestamptz), + ( + "c", + DataType::Struct(StructType::new(vec![ + ("aa", DataType::Int64), + ("bb", DataType::Float64), + ])), + ), + ])), + name: "v10".into(), + sub_fields: vec![ + Field { + data_type: DataType::Timestamp, + name: "a".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Timestamptz, + name: "b".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Struct(StructType::new(vec![ + ("aa", DataType::Int64), + ("bb", DataType::Float64), + ])), + name: "c".into(), + sub_fields: vec![ + Field { + data_type: DataType::Int64, + name: "aa".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Float64, + name: "bb".into(), + ..mock_field.clone() + }, + ], + ..mock_field.clone() + }, + ], + ..mock_field.clone() + }, + Field { + data_type: DataType::List(Box::new(DataType::List(Box::new(DataType::Struct( + StructType::new(vec![("aa", DataType::Int64), ("bb", DataType::Float64)]), + ))))), + name: "v11".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Jsonb, + name: "12".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Serial, + name: "13".into(), + ..mock_field.clone() + }, + Field { + data_type: DataType::Int256, + name: "14".into(), + ..mock_field.clone() + }, + ]; + let schema = json_converter_with_schema(json!({}), "test".to_owned(), fields.iter()) + ["schema"] + .to_string(); + let ans = r#"{"fields":[{"field":"v1","optional":"true","type":"boolean"},{"field":"v2","optional":"true","type":"int16"},{"field":"v3","optional":"true","type":"int32"},{"field":"v4","optional":"true","type":"float"},{"field":"v5","optional":"true","type":"string"},{"field":"v6","optional":"true","type":"int32"},{"field":"v7","optional":"true","type":"string"},{"field":"v8","optional":"true","type":"int64"},{"field":"v9","optional":"true","type":"string"},{"field":"v10","fields":[{"field":"a","optional":"true","type":"int64"},{"field":"b","optional":"true","type":"string"},{"field":"c","fields":[{"field":"aa","optional":"true","type":"int64"},{"field":"bb","optional":"true","type":"double"}],"optional":"true","type":"struct"}],"optional":"true","type":"struct"},{"field":"v11","items":{"items":{"fields":[{"field":"aa","optional":"true","type":"int64"},{"field":"bb","optional":"true","type":"double"}],"optional":"true","type":"struct"},"optional":"true","type":"array"},"optional":"true","type":"array"},{"field":"12","optional":"true","type":"string"},{"field":"13","optional":"true","type":"int32"},{"field":"14","optional":"true","type":"string"}],"name":"test","optional":"false","type":"struct"}"#; + assert_eq!(schema, ans); + } } diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 1865461a65e5..682907db447f 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -115,20 +115,20 @@ impl SinkFormatterImpl { let mut val_encoder = JsonEncoder::new(schema, None, TimestampHandlingMode::Milli); if let Some(s) = format_desc.options.get("schemas.enable") { - match s.to_lowercase().as_str() { - "true" => { + match s.to_lowercase().parse::() { + Ok(true) => { let kafka_connect = KafkaConnectParams { schema_name: format!("{}.{}", db_name, sink_from_name), }; key_encoder = key_encoder.with_kafka_connect(kafka_connect.clone()); val_encoder = val_encoder.with_kafka_connect(kafka_connect); } - "false" => {} - other => { + Ok(false) => {} + _ => { return Err(SinkError::Config(anyhow!( "schemas.enable is expected to be `true` or `false`, got {}", - other - ))) + s + ))); } } }; From c9fc80e7e6e08195bed85aac302a9ea5880f2625 Mon Sep 17 00:00:00 2001 From: Rossil <40714231+Rossil2012@users.noreply.github.com> Date: Thu, 19 Oct 2023 10:38:36 +0800 Subject: [PATCH 21/21] fix optional's json type --- e2e_test/sink/kafka/upsert_schema1.result | 20 +++++++++---------- e2e_test/sink/kafka/upsert_schema2.result | 22 ++++++++++----------- e2e_test/sink/kafka/upsert_schema3.result | 24 +++++++++++------------ src/connector/src/sink/encoder/json.rs | 23 +++++++++++----------- src/connector/src/sink/encoder/mod.rs | 3 +++ src/connector/src/sink/formatter/mod.rs | 2 +- 6 files changed, 48 insertions(+), 46 deletions(-) diff --git a/e2e_test/sink/kafka/upsert_schema1.result b/e2e_test/sink/kafka/upsert_schema1.result index 2af049c82626..d33394de4e5b 100644 --- a/e2e_test/sink/kafka/upsert_schema1.result +++ b/e2e_test/sink/kafka/upsert_schema1.result @@ -1,10 +1,10 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema2.result b/e2e_test/sink/kafka/upsert_schema2.result index 1231414b5f7b..e560df682107 100644 --- a/e2e_test/sink/kafka/upsert_schema2.result +++ b/e2e_test/sink/kafka/upsert_schema2.result @@ -1,11 +1,11 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} diff --git a/e2e_test/sink/kafka/upsert_schema3.result b/e2e_test/sink/kafka/upsert_schema3.result index 7a32af185f63..85c3db9768e7 100644 --- a/e2e_test/sink/kafka/upsert_schema3.result +++ b/e2e_test/sink/kafka/upsert_schema3.result @@ -1,12 +1,12 @@ -{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} null -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} -{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":"true","type":"int32"},{"field":"v_varchar","optional":"true","type":"string"},{"field":"v_smallint","optional":"true","type":"int16"},{"field":"v_integer","optional":"true","type":"int32"},{"field":"v_bigint","optional":"true","type":"int64"},{"field":"v_float","optional":"true","type":"float"},{"field":"v_double","optional":"true","type":"double"},{"field":"v_timestamp","optional":"true","type":"int64"}],"name":"dev.t_kafka","optional":"false","type":"struct"}} +{"payload":{"id":10},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":10,"v_bigint":20674,"v_double":9042.404483827513,"v_float":19387.23828125,"v_integer":20674,"v_smallint":26951,"v_timestamp":1681404058888,"v_varchar":"0oVqRIHqkb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} null +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":0,"v_double":0.0,"v_float":0.0,"v_integer":0,"v_smallint":0,"v_timestamp":0,"v_varchar":""},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":1},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":1,"v_bigint":1872,"v_double":23956.39329760601,"v_float":26261.416015625,"v_integer":1872,"v_smallint":31031,"v_timestamp":1681453634104,"v_varchar":"8DfUFencLe"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":2},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":2,"v_bigint":4598,"v_double":31923.077305746086,"v_float":27031.224609375,"v_integer":4598,"v_smallint":22690,"v_timestamp":1681429444869,"v_varchar":"sIo1XXVeHZ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":3},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":3,"v_bigint":14894,"v_double":9742.475509566086,"v_float":2660.290283203125,"v_integer":5894,"v_smallint":5985,"v_timestamp":1681429011269,"v_varchar":"LVLAhd1pQv"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":4},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":4,"v_bigint":24962,"v_double":3119.719721891862,"v_float":21217.77734375,"v_integer":7406,"v_smallint":6306,"v_timestamp":1681434727993,"v_varchar":"ORjwy3oMNb"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":5},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":5,"v_bigint":9253,"v_double":17464.91553421121,"v_float":22749.5,"v_integer":9253,"v_smallint":22765,"v_timestamp":1681444642324,"v_varchar":"sSkKswxrYd"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":6},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":6,"v_bigint":28842,"v_double":11210.458724794062,"v_float":5885.3681640625,"v_integer":10844,"v_smallint":4014,"v_timestamp":1681382522137,"v_varchar":"V4y71v4Gip"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":7},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":7,"v_bigint":15914,"v_double":10967.182297153104,"v_float":3946.743408203125,"v_integer":12652,"v_smallint":10324,"v_timestamp":1681447263083,"v_varchar":"YIVLnWxHyf"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":8},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":8,"v_bigint":28641,"v_double":993.408963466774,"v_float":13652.0732421875,"v_integer":19036,"v_smallint":194,"v_timestamp":1681393929356,"v_varchar":"lv7Eq3g8hx"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} +{"payload":{"id":9},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} {"payload":{"id":9,"v_bigint":24837,"v_double":11615.276406159757,"v_float":20699.55859375,"v_integer":20090,"v_smallint":10028,"v_timestamp":1681389642487,"v_varchar":"nwRq4zejSQ"},"schema":{"fields":[{"field":"id","optional":true,"type":"int32"},{"field":"v_varchar","optional":true,"type":"string"},{"field":"v_smallint","optional":true,"type":"int16"},{"field":"v_integer","optional":true,"type":"int32"},{"field":"v_bigint","optional":true,"type":"int64"},{"field":"v_float","optional":true,"type":"float"},{"field":"v_double","optional":true,"type":"double"},{"field":"v_timestamp","optional":true,"type":"int64"}],"name":"dev.t_kafka","optional":false,"type":"struct"}} diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index 636cb6f4199d..12176b491d6a 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::sync::Arc; use base64::engine::general_purpose; use base64::Engine as _; @@ -26,7 +27,10 @@ use risingwave_common::types::{DataType, DatumRef, Decimal, ScalarRefImpl, ToTex use risingwave_common::util::iter_util::ZipEqDebug; use serde_json::{json, Map, Value}; -use super::{CustomJsonType, KafkaConnectParams, Result, RowEncoder, SerTo, TimestampHandlingMode}; +use super::{ + CustomJsonType, KafkaConnectParams, KafkaConnectParamsRef, Result, RowEncoder, SerTo, + TimestampHandlingMode, +}; use crate::sink::SinkError; pub struct JsonEncoder { @@ -34,7 +38,7 @@ pub struct JsonEncoder { col_indices: Option>, timestamp_handling_mode: TimestampHandlingMode, custom_json_type: CustomJsonType, - kafka_connect: Option, + kafka_connect: Option, } impl JsonEncoder { @@ -69,7 +73,7 @@ impl JsonEncoder { pub fn with_kafka_connect(self, kafka_connect: KafkaConnectParams) -> Self { Self { - kafka_connect: Some(kafka_connect), + kafka_connect: Some(Arc::new(kafka_connect)), ..self } } @@ -130,7 +134,7 @@ impl SerTo for Value { } } -pub fn datum_to_json_object( +fn datum_to_json_object( field: &Field, datum: DatumRef<'_>, timestamp_handling_mode: TimestampHandlingMode, @@ -298,7 +302,7 @@ fn json_converter_with_schema<'a>( mapping.insert("field".to_string(), json!(field.name)); mapping }).collect_vec(), - "optional": "false", + "optional": false, "name": name, }), ); @@ -334,7 +338,7 @@ pub(crate) fn schema_type_mapping(rw_type: &DataType) -> &'static str { fn type_as_json_schema(rw_type: &DataType) -> Map { let mut mapping = Map::with_capacity(4); // type + optional + fields/items + field mapping.insert("type".to_string(), json!(schema_type_mapping(rw_type))); - mapping.insert("optional".to_string(), json!("true")); + mapping.insert("optional".to_string(), json!(true)); match rw_type { DataType::Struct(struct_type) => { let sub_fields = struct_type @@ -356,11 +360,6 @@ fn type_as_json_schema(rw_type: &DataType) -> Map { mapping } -// reference: https://github.com/apache/kafka/blob/80982c4ae3fe6be127b48ec09caff11ab5f87c69/connect/json/src/main/java/org/apache/kafka/connect/json/JsonConverter.java#L356 -fn field_as_json_schema(field: &Field) -> Value { - Value::Object(type_as_json_schema(&field.data_type)) -} - #[cfg(test)] mod tests { @@ -670,7 +669,7 @@ mod tests { let schema = json_converter_with_schema(json!({}), "test".to_owned(), fields.iter()) ["schema"] .to_string(); - let ans = r#"{"fields":[{"field":"v1","optional":"true","type":"boolean"},{"field":"v2","optional":"true","type":"int16"},{"field":"v3","optional":"true","type":"int32"},{"field":"v4","optional":"true","type":"float"},{"field":"v5","optional":"true","type":"string"},{"field":"v6","optional":"true","type":"int32"},{"field":"v7","optional":"true","type":"string"},{"field":"v8","optional":"true","type":"int64"},{"field":"v9","optional":"true","type":"string"},{"field":"v10","fields":[{"field":"a","optional":"true","type":"int64"},{"field":"b","optional":"true","type":"string"},{"field":"c","fields":[{"field":"aa","optional":"true","type":"int64"},{"field":"bb","optional":"true","type":"double"}],"optional":"true","type":"struct"}],"optional":"true","type":"struct"},{"field":"v11","items":{"items":{"fields":[{"field":"aa","optional":"true","type":"int64"},{"field":"bb","optional":"true","type":"double"}],"optional":"true","type":"struct"},"optional":"true","type":"array"},"optional":"true","type":"array"},{"field":"12","optional":"true","type":"string"},{"field":"13","optional":"true","type":"int32"},{"field":"14","optional":"true","type":"string"}],"name":"test","optional":"false","type":"struct"}"#; + let ans = r#"{"fields":[{"field":"v1","optional":true,"type":"boolean"},{"field":"v2","optional":true,"type":"int16"},{"field":"v3","optional":true,"type":"int32"},{"field":"v4","optional":true,"type":"float"},{"field":"v5","optional":true,"type":"string"},{"field":"v6","optional":true,"type":"int32"},{"field":"v7","optional":true,"type":"string"},{"field":"v8","optional":true,"type":"int64"},{"field":"v9","optional":true,"type":"string"},{"field":"v10","fields":[{"field":"a","optional":true,"type":"int64"},{"field":"b","optional":true,"type":"string"},{"field":"c","fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"}],"optional":true,"type":"struct"},{"field":"v11","items":{"items":{"fields":[{"field":"aa","optional":true,"type":"int64"},{"field":"bb","optional":true,"type":"double"}],"optional":true,"type":"struct"},"optional":true,"type":"array"},"optional":true,"type":"array"},{"field":"12","optional":true,"type":"string"},{"field":"13","optional":true,"type":"int32"},{"field":"14","optional":true,"type":"string"}],"name":"test","optional":false,"type":"struct"}"#; assert_eq!(schema, ans); } } diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs index 389359a2284e..6a2558f5fc35 100644 --- a/src/connector/src/sink/encoder/mod.rs +++ b/src/connector/src/sink/encoder/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. use std::collections::HashMap; +use std::sync::Arc; use risingwave_common::catalog::Schema; use risingwave_common::row::Row; @@ -136,3 +137,5 @@ impl From for super::SinkError { pub struct KafkaConnectParams { pub schema_name: String, } + +type KafkaConnectParamsRef = Arc; diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 682907db447f..44c8ed6f3ea2 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -18,7 +18,7 @@ use risingwave_common::array::StreamChunk; use crate::sink::{Result, SinkError}; mod append_only; -pub mod debezium_json; +mod debezium_json; mod upsert; pub use append_only::AppendOnlyFormatter;