From 6e0149e05d0acec3f2fac00d02cef2f06486e7f0 Mon Sep 17 00:00:00 2001 From: Kai Jellinghaus Date: Thu, 18 Jan 2024 21:07:46 +0100 Subject: [PATCH 1/5] Support Inserting Enum16 Ids --- src/connector/src/sink/clickhouse.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/connector/src/sink/clickhouse.rs b/src/connector/src/sink/clickhouse.rs index 123739f4a5618..2a279e6f5e4ab 100644 --- a/src/connector/src/sink/clickhouse.rs +++ b/src/connector/src/sink/clickhouse.rs @@ -260,9 +260,9 @@ impl ClickHouseSink { ) -> Result<()> { let is_match = match fields_type { risingwave_common::types::DataType::Boolean => Ok(ck_column.r#type.contains("Bool")), - risingwave_common::types::DataType::Int16 => { - Ok(ck_column.r#type.contains("UInt16") | ck_column.r#type.contains("Int16")) - } + risingwave_common::types::DataType::Int16 => Ok(ck_column.r#type.contains("UInt16") + | ck_column.r#type.contains("Int16") + | ck_column.r#type.contains("Enum16")), risingwave_common::types::DataType::Int32 => { Ok(ck_column.r#type.contains("UInt32") | ck_column.r#type.contains("Int32")) } From 57059319316e003d1147e89bdf075da5aadff68f Mon Sep 17 00:00:00 2001 From: Kai Jellinghaus Date: Sat, 20 Jan 2024 11:47:03 +0100 Subject: [PATCH 2/5] Clickhouse Enum E2E --- ci/scripts/e2e-clickhouse-sink-test.sh | 16 ++++++++-------- e2e_test/sink/clickhouse_sink.slt | 6 +++--- src/connector/src/sink/clickhouse.rs | 2 ++ 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ci/scripts/e2e-clickhouse-sink-test.sh b/ci/scripts/e2e-clickhouse-sink-test.sh index 51720b0e1d09b..1e04cacc0473b 100755 --- a/ci/scripts/e2e-clickhouse-sink-test.sh +++ b/ci/scripts/e2e-clickhouse-sink-test.sh @@ -31,7 +31,7 @@ sleep 1 echo "--- create clickhouse table" curl https://clickhouse.com/ | sh sleep 2 -./clickhouse client --host=clickhouse-server --port=9000 --query="CREATE table demo_test(v1 Int32,v2 Int64,v3 String)ENGINE = ReplacingMergeTree PRIMARY KEY (v1);" +./clickhouse client --host=clickhouse-server --port=9000 --query="CREATE table demo_test(v1 Int32,v2 Int64,v3 String,v4 Enum16('A'=1,'B'=2))ENGINE = ReplacingMergeTree PRIMARY KEY (v1);" echo "--- testing sinks" sqllogictest -p 4566 -d dev './e2e_test/sink/clickhouse_sink.slt' @@ -41,13 +41,13 @@ sleep 1 # check sink destination using shell if cat ./query_result.csv | sort | awk -F "," '{ -if ($1 == 1 && $2 == 50 && $3 == "\"1-50\"") c1++; - if ($1 == 13 && $2 == 2 && $3 == "\"13-2\"") c2++; - if ($1 == 2 && $2 == 2 && $3 == "\"2-2\"") c3++; - if ($1 == 21 && $2 == 2 && $3 == "\"21-2\"") c4++; - if ($1 == 3 && $2 == 2 && $3 == "\"3-2\"") c5++; - if ($1 == 5 && $2 == 2 && $3 == "\"5-2\"") c6++; - if ($1 == 8 && $2 == 2 && $3 == "\"8-2\"") c7++; } +if ($1 == 1 && $2 == 50 && $3 == "\"1-50\"" && $4 == "\"A\"") c1++; + if ($1 == 13 && $2 == 2 && $3 == "\"13-2\"" && $4 == "\"B\"") c2++; + if ($1 == 2 && $2 == 2 && $3 == "\"2-2\"" && $4 == "\"A\"") c3++; + if ($1 == 21 && $2 == 2 && $3 == "\"21-2\"" && $4 == "\"B\"") c4++; + if ($1 == 3 && $2 == 2 && $3 == "\"3-2\"" && $4 == "\"A\"") c5++; + if ($1 == 5 && $2 == 2 && $3 == "\"5-2\"" && $4 == "\"B\"") c6++; + if ($1 == 8 && $2 == 2 && $3 == "\"8-2\"" && $4 == "\"A\"") c7++; } END { exit !(c1 == 1 && c2 == 1 && c3 == 1 && c4 == 1 && c5 == 1 && c6 == 1 && c7 == 1); }'; then echo "Clickhouse sink check passed" else diff --git a/e2e_test/sink/clickhouse_sink.slt b/e2e_test/sink/clickhouse_sink.slt index 909bdbfd6356b..8ac1d046db824 100644 --- a/e2e_test/sink/clickhouse_sink.slt +++ b/e2e_test/sink/clickhouse_sink.slt @@ -1,11 +1,11 @@ statement ok -CREATE TABLE t6 (v1 int primary key, v2 bigint, v3 varchar); +CREATE TABLE t6 (v1 int primary key, v2 bigint, v3 varchar, v4 smallint); statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH ( +CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3, m6.v4 as v4 from mv6 WITH ( connector = 'clickhouse', type = 'append-only', force_append_only='true', @@ -17,7 +17,7 @@ CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH ); statement ok -INSERT INTO t6 VALUES (1, 50, '1-50'), (2, 2, '2-2'), (3, 2, '3-2'), (5, 2, '5-2'), (8, 2, '8-2'), (13, 2, '13-2'), (21, 2, '21-2'); +INSERT INTO t6 VALUES (1, 50, '1-50', 1), (2, 2, '2-2', 2), (3, 2, '3-2', 1), (5, 2, '5-2', 2), (8, 2, '8-2', 1), (13, 2, '13-2', 2), (21, 2, '21-2', 1); statement ok FLUSH; diff --git a/src/connector/src/sink/clickhouse.rs b/src/connector/src/sink/clickhouse.rs index 2a279e6f5e4ab..5157006493daa 100644 --- a/src/connector/src/sink/clickhouse.rs +++ b/src/connector/src/sink/clickhouse.rs @@ -262,6 +262,8 @@ impl ClickHouseSink { risingwave_common::types::DataType::Boolean => Ok(ck_column.r#type.contains("Bool")), risingwave_common::types::DataType::Int16 => Ok(ck_column.r#type.contains("UInt16") | ck_column.r#type.contains("Int16") + // Allow Int16 to be pushed to Enum16, they share an encoding and value range + // No special care is taken to ensure values are valid. | ck_column.r#type.contains("Enum16")), risingwave_common::types::DataType::Int32 => { Ok(ck_column.r#type.contains("UInt32") | ck_column.r#type.contains("Int32")) From 8791c65902facf23178e3b66b8c6565618163410 Mon Sep 17 00:00:00 2001 From: Kai Jellinghaus Date: Sat, 20 Jan 2024 11:59:04 +0100 Subject: [PATCH 3/5] Fix Trailing Space --- src/connector/src/sink/clickhouse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/src/sink/clickhouse.rs b/src/connector/src/sink/clickhouse.rs index 5157006493daa..e29ad2dd89775 100644 --- a/src/connector/src/sink/clickhouse.rs +++ b/src/connector/src/sink/clickhouse.rs @@ -263,7 +263,7 @@ impl ClickHouseSink { risingwave_common::types::DataType::Int16 => Ok(ck_column.r#type.contains("UInt16") | ck_column.r#type.contains("Int16") // Allow Int16 to be pushed to Enum16, they share an encoding and value range - // No special care is taken to ensure values are valid. + // No special care is taken to ensure values are valid. | ck_column.r#type.contains("Enum16")), risingwave_common::types::DataType::Int32 => { Ok(ck_column.r#type.contains("UInt32") | ck_column.r#type.contains("Int32")) From 7927d89794e9d6c72d2fdc6537ee4eeee43e41c3 Mon Sep 17 00:00:00 2001 From: Kai Jellinghaus Date: Tue, 23 Jan 2024 20:00:02 +0100 Subject: [PATCH 4/5] Fix Typo --- e2e_test/sink/clickhouse_sink.slt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e_test/sink/clickhouse_sink.slt b/e2e_test/sink/clickhouse_sink.slt index 8ac1d046db824..9791f484326d7 100644 --- a/e2e_test/sink/clickhouse_sink.slt +++ b/e2e_test/sink/clickhouse_sink.slt @@ -5,7 +5,7 @@ statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3, m6.v4 as v4 from mv6 WITH ( +CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3, mv6.v4 as v4 from mv6 WITH ( connector = 'clickhouse', type = 'append-only', force_append_only='true', From a5c2b075e4efab1ac336536ec02ec7465dd559cc Mon Sep 17 00:00:00 2001 From: Kai Jellinghaus Date: Fri, 9 Feb 2024 10:27:05 +0100 Subject: [PATCH 5/5] Fix Enum Check ordering --- ci/scripts/e2e-clickhouse-sink-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scripts/e2e-clickhouse-sink-test.sh b/ci/scripts/e2e-clickhouse-sink-test.sh index 1e04cacc0473b..784c1b74c874b 100755 --- a/ci/scripts/e2e-clickhouse-sink-test.sh +++ b/ci/scripts/e2e-clickhouse-sink-test.sh @@ -43,8 +43,8 @@ sleep 1 if cat ./query_result.csv | sort | awk -F "," '{ if ($1 == 1 && $2 == 50 && $3 == "\"1-50\"" && $4 == "\"A\"") c1++; if ($1 == 13 && $2 == 2 && $3 == "\"13-2\"" && $4 == "\"B\"") c2++; - if ($1 == 2 && $2 == 2 && $3 == "\"2-2\"" && $4 == "\"A\"") c3++; - if ($1 == 21 && $2 == 2 && $3 == "\"21-2\"" && $4 == "\"B\"") c4++; + if ($1 == 2 && $2 == 2 && $3 == "\"2-2\"" && $4 == "\"B\"") c3++; + if ($1 == 21 && $2 == 2 && $3 == "\"21-2\"" && $4 == "\"A\"") c4++; if ($1 == 3 && $2 == 2 && $3 == "\"3-2\"" && $4 == "\"A\"") c5++; if ($1 == 5 && $2 == 2 && $3 == "\"5-2\"" && $4 == "\"B\"") c6++; if ($1 == 8 && $2 == 2 && $3 == "\"8-2\"" && $4 == "\"A\"") c7++; }