test(mssql-cdc): enrich tests and small refactor (#18081)

risingwavelabs · Aug 21, 2024 · a4a4583 · a4a4583
1 parent da46c4d
commit a4a4583
Show file tree

Hide file tree

Showing 6 changed files with 234 additions and 123 deletions.
diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt
@@ -254,11 +254,17 @@ CREATE TABLE shared_orders (
 )  from mssql_source table 'dbo.orders';
 
 statement ok
-CREATE TABLE shared_single_type (
-    id INT,
-    c_time time,
-    PRIMARY KEY (id)
-) from mssql_source table 'dbo.single_type';
+CREATE TABLE shared_single_type (*) from mssql_source table 'dbo.single_type';
+
+statement ok
+CREATE TABLE shared_sqlserver_all_data_types (
+    *
+) from mssql_source table 'dbo.sqlserver_all_data_types';
+
+sleep 5s
+
+statement ok
+DROP table shared_sqlserver_all_data_types;
 
 statement ok
 CREATE TABLE shared_sqlserver_all_data_types (
@@ -271,12 +277,18 @@ CREATE TABLE shared_sqlserver_all_data_types (
     c_decimal DECIMAL,
     c_real REAL,
     c_float FLOAT,
+    c_char VARCHAR,
     c_varchar VARCHAR,
+    c_nvarchar VARCHAR,
+    c_ntext VARCHAR,
+    c_binary BYTEA,
     c_varbinary BYTEA,
+    c_uniqueidentifier VARCHAR,
     c_date DATE,
     c_time TIME,
     c_datetime2 TIMESTAMP,
     c_datetimeoffset TIMESTAMPTZ,
+    c_xml varchar,
     PRIMARY KEY (id)
 ) from mssql_source table 'dbo.sqlserver_all_data_types';
 
@@ -306,7 +318,12 @@ CREATE TABLE upper_table (
     "ID" INT,
     "Name" VARCHAR,
     PRIMARY KEY ("ID")
-)  from upper_mssql_source table 'UpperSchema.UpperTable';
+)
+INCLUDE TIMESTAMP AS commit_ts
+INCLUDE DATABASE_NAME as database_name
+INCLUDE SCHEMA_NAME as schema_name
+INCLUDE TABLE_NAME as table_name
+from upper_mssql_source table 'UpperSchema.UpperTable';
 
 statement ok
 create materialized view shared_orders_cnt as select count(*) as cnt from shared_orders;
@@ -349,11 +366,46 @@ SELECT * from shared_single_type order by id;
 3 23:59:59.999
 
 query TTTTTTT
-SELECT * from shared_sqlserver_all_data_types order by id;
+SELECT id, c_bit, c_tinyint, c_smallint, c_int, c_bigint from shared_sqlserver_all_data_types order by id;
+----
+1 f 0 0 0 0
+2 t 255 -32768 -2147483648 -9223372036854775808
+3 t 127 32767 2147483647 9223372036854775807
+
+query TTTT
+SELECT id, c_decimal, c_real, c_float from shared_sqlserver_all_data_types order by id;
+----
+1 0 0 0
+2 -10 -10000 -10000
+3 -10 10000 10000
+
+query TTTTTTT
+SELECT id, c_varchar, c_nvarchar, c_ntext, c_binary, c_varbinary, c_uniqueidentifier from shared_sqlserver_all_data_types order by id;
+----
+1 (empty) 中 中 \xff000000 NULL NULL
+2 aa 🌹 🌹 NULL \xff 6F9619FF-8B86-D011-B42D-00C04FC964FF
+3 zzzz 🌹👍 🌹👍 \xffffffff \xffffffff 6F9619FF-8B86-D011-B42D-00C04FC964FF
+
+query TTT
+SELECT id, char_length(c_char) from shared_sqlserver_all_data_types order by id;
+----
+1 4
+2 4
+3 4
+
+query TTTTTTT
+SELECT id, c_date, c_time, c_datetime2, c_datetimeoffset from shared_sqlserver_all_data_types order by id;
 ----
-1 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
-2 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
-3 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
+1 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
+2 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
+3 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
+
+query TTTTTTT
+SELECT id, c_xml from shared_sqlserver_all_data_types order by id;
+----
+1 <Person><Name>John Doe</Name><Age>30</Age></Person>
+2 <Person><Name>Jane Doe</Name><Age>28</Age></Person>
+3 <Name>Jane Doe</Name>
 
 # ------------ kill cluster ------------
 # system ok
@@ -410,20 +462,80 @@ SELECT * from shared_single_type order by id;
 13 23:59:59.999
 
 query TTTTTTT
-SELECT * from shared_sqlserver_all_data_types order by id;
+SELECT id, c_bit, c_tinyint, c_smallint, c_int, c_bigint from shared_sqlserver_all_data_types order by id;
+----
+1 f 0 0 0 0
+2 t 255 -32768 -2147483648 -9223372036854775808
+3 t 127 32767 2147483647 9223372036854775807
+11 f 0 0 0 0
+12 t 255 -32768 -2147483648 -9223372036854775808
+13 t 127 32767 2147483647 9223372036854775807
+
+query TTTT
+SELECT id, c_decimal, c_real, c_float from shared_sqlserver_all_data_types order by id;
+----
+1 0 0 0
+2 -10 -10000 -10000
+3 -10 10000 10000
+11 0 0 0
+12 -10 -10000 -10000
+13 -10 10000 10000
+
+query TTTTTTT
+SELECT id, c_varchar, c_nvarchar, c_ntext, c_binary, c_varbinary, c_uniqueidentifier from shared_sqlserver_all_data_types order by id;
 ----
-1 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
-2 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
-3 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
-11 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
-12 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
-13 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
-
-query TT
-SELECT * from upper_table order by "ID";
+1 (empty) 中 中 \xff000000 NULL NULL
+2 aa 🌹 🌹 NULL \xff 6F9619FF-8B86-D011-B42D-00C04FC964FF
+3 zzzz 🌹👍 🌹👍 \xffffffff \xffffffff 6F9619FF-8B86-D011-B42D-00C04FC964FF
+11 (empty) 中 中 \xff000000 NULL NULL
+12 aa 🌹 🌹 NULL \xff 6F9619FF-8B86-D011-B42D-00C04FC964FF
+13 zzzz 🌹👍 🌹👍 \xffffffff \xffffffff 6F9619FF-8B86-D011-B42D-00C04FC964FF
+
+query TTT
+SELECT id, char_length(c_char) from shared_sqlserver_all_data_types order by id;
+----
+1 4
+2 4
+3 4
+11 4
+12 4
+13 4
+
+query TTTTTTT
+SELECT id, c_date, c_time, c_datetime2, c_datetimeoffset from shared_sqlserver_all_data_types order by id;
+----
+1 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
+2 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
+3 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
+11 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
+12 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
+13 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
+
+query TTTTTTT
+SELECT id, c_xml from shared_sqlserver_all_data_types order by id;
+----
+1 <Person><Name>John Doe</Name><Age>30</Age></Person>
+2 <Person><Name>Jane Doe</Name><Age>28</Age></Person>
+3 <Name>Jane Doe</Name>
+11 <Person><Name>John Doe</Name><Age>30</Age></Person>
+12 <Person><Name>Jane Doe</Name><Age>28</Age></Person>
+13 <Name>Jane Doe</Name>
+
+query TTTTTTTT
+SELECT "ID", "Name", database_name, schema_name, table_name from upper_table order by "ID";
+----
+1 Alice UpperDB UpperSchema UpperTable
+11 Alice UpperDB UpperSchema UpperTable
+
+query TTTTTTTT
+SELECT commit_ts from upper_table where "ID" = 1;
+----
+1970-01-01 00:00:00+00:00
+
+query TTTTTTTT
+SELECT commit_ts > '2024-08-16 00:00:00.000+00:00' from upper_table where "ID" = 11;
 ----
-1  Alice
-11 Alice
+t
 
 # ------------ drop stage ------------
 statement ok

diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql
@@ -16,9 +16,8 @@ VALUES
 
 INSERT INTO single_type VALUES (13, '23:59:59.999')
 
+INSERT INTO sqlserver_all_data_types VALUES (11, 'False', 0, 0, 0, 0, 0, 0, 0, '', '', N'中', N'中', 0xff, NULL, NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00', '<Person><Name>John Doe</Name><Age>30</Age></Person>');
 
-INSERT INTO sqlserver_all_data_types VALUES (11, 'False', 0, 0, 0, 0, 0, 0, 0, '', NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00');
+INSERT INTO sqlserver_all_data_types VALUES (12, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 'aa', N'🌹', N'🌹', NULL, 0xff, '6f9619ff-8b86-d011-b42d-00c04fc964ff', '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123', '<Person> <Name>Jane Doe</Name> <Age>28</Age> </Person>');
 
-INSERT INTO sqlserver_all_data_types VALUES (12, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 0xff, '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123');
-
-INSERT INTO sqlserver_all_data_types VALUES (13, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 0xffffffff, '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999')
+INSERT INTO sqlserver_all_data_types VALUES (13, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 'zzzz', N'🌹👍', N'🌹👍', 0xffffffff, 0xffffffff, '6F9619FF-8B86-D011-B42D-00C04FC964FF', '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999', '<Name>Jane Doe</Name>')
diff --git a/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql b/e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql
@@ -52,24 +52,30 @@ CREATE TABLE sqlserver_all_data_types (
   c_decimal DECIMAL(28),
   c_real real,
   c_float float,
+  c_char char(4),
   c_varchar varchar(4),
+  c_nvarchar nvarchar(4),
+  c_ntext ntext,
+  c_binary binary(4),
   c_varbinary varbinary(4),
+  c_uniqueidentifier uniqueidentifier,
   c_date date,
   c_time time,
   c_datetime2 datetime2,
-  c_datetimeoffset datetimeoffset
+  c_datetimeoffset datetimeoffset,
+  c_xml xml
 );
 
 EXEC sys.sp_cdc_enable_table
   @source_schema = 'dbo',
   @source_name = 'sqlserver_all_data_types',
   @role_name = NULL;
 
-INSERT INTO sqlserver_all_data_types VALUES (1, 'False', 0, 0, 0, 0, 0, 0, 0, '', NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00');
+INSERT INTO sqlserver_all_data_types VALUES (1, 'False', 0, 0, 0, 0, 0, 0, 0, '', '', N'中', N'中', 0xff, NULL, NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00', '<Person><Name>John Doe</Name><Age>30</Age></Person>');
 
-INSERT INTO sqlserver_all_data_types VALUES (2, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 0xff, '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123');
+INSERT INTO sqlserver_all_data_types VALUES (2, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 'aa', N'🌹', N'🌹', NULL, 0xff, '6f9619ff-8b86-d011-b42d-00c04fc964ff', '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123', '<Person> <Name>Jane Doe</Name> <Age>28</Age> </Person>');
 
-INSERT INTO sqlserver_all_data_types VALUES (3, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 0xffffffff, '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999')
+INSERT INTO sqlserver_all_data_types VALUES (3, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 'zzzz', N'🌹👍', N'🌹👍', 0xffffffff, 0xffffffff, '6F9619FF-8B86-D011-B42D-00C04FC964FF', '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999', '<Name>Jane Doe</Name>')
 
 -- Table without enabling CDC
 CREATE TABLE orders_without_cdc (

diff --git a/...ctor-service/src/main/java/com/risingwave/connector/source/common/SqlServerValidator.java b/...ctor-service/src/main/java/com/risingwave/connector/source/common/SqlServerValidator.java
@@ -289,6 +289,7 @@ private boolean isDataTypeCompatible(String ssDataType, Data.DataType.TypeName t
                 return Data.DataType.TypeName.INT16_VALUE <= val
                         && val <= Data.DataType.TypeName.INT64_VALUE;
             case "integer":
+            case "int":
                 return Data.DataType.TypeName.INT32_VALUE <= val
                         && val <= Data.DataType.TypeName.INT64_VALUE;
             case "bigint":
@@ -305,9 +306,15 @@ private boolean isDataTypeCompatible(String ssDataType, Data.DataType.TypeName t
             case "decimal":
             case "numeric":
                 return val == Data.DataType.TypeName.DECIMAL_VALUE;
+            case "char":
+            case "nchar":
             case "varchar":
-            case "character varying":
+            case "nvarchar":
+            case "text":
+            case "ntext":
+            case "uniqueidentifier":
                 return val == Data.DataType.TypeName.VARCHAR_VALUE;
+            case "binary":
             case "varbinary":
                 return val == Data.DataType.TypeName.BYTEA_VALUE;
             case "date":