Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test(mssql-cdc): enrich tests and small refactor #18081

Merged
merged 10 commits into from
Aug 21, 2024
Merged

test(mssql-cdc): enrich tests and small refactor #18081

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
34ee05d
test(mssql-cdc): enrich tests and small refactor
KeXiangWang Aug 18, 2024
1aa2958
add uuid
KeXiangWang Aug 18, 2024
cdfddb5
add binary test
KeXiangWang Aug 18, 2024
ce936e9
add more test
KeXiangWang Aug 18, 2024
f1d6a3f
fix
KeXiangWang Aug 19, 2024
e24e9e9
tmp
KeXiangWang Aug 19, 2024
a6c5818
tmp
KeXiangWang Aug 19, 2024
019e288
fix
KeXiangWang Aug 19, 2024
7a2db04
fix
KeXiangWang Aug 19, 2024
41a852b
Merge branch 'main' into wkx/enrich-sql-server-cdc-test
KeXiangWang Aug 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
156 changes: 134 additions & 22 deletions e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,11 +254,17 @@ CREATE TABLE shared_orders (
) from mssql_source table 'dbo.orders';

statement ok
CREATE TABLE shared_single_type (
id INT,
c_time time,
PRIMARY KEY (id)
) from mssql_source table 'dbo.single_type';
CREATE TABLE shared_single_type (*) from mssql_source table 'dbo.single_type';

statement ok
CREATE TABLE shared_sqlserver_all_data_types (
*
) from mssql_source table 'dbo.sqlserver_all_data_types';

sleep 5s
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What's the intention of the sleep here? create - sleep - drop looks a little weird.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously, I found if the types found by (*) are wrongly converted to RW types, RW will crash explicitly when consuming the data in the upstream table. I've fixed the found mismatch in this PR. This test sleep 5s is to ensure the data is consumed and the mismatch is encountered so that it can be exposed.


statement ok
DROP table shared_sqlserver_all_data_types;

statement ok
CREATE TABLE shared_sqlserver_all_data_types (
Expand All @@ -271,12 +277,18 @@ CREATE TABLE shared_sqlserver_all_data_types (
c_decimal DECIMAL,
c_real REAL,
c_float FLOAT,
c_char VARCHAR,
c_varchar VARCHAR,
c_nvarchar VARCHAR,
c_ntext VARCHAR,
c_binary BYTEA,
c_varbinary BYTEA,
c_uniqueidentifier VARCHAR,
c_date DATE,
c_time TIME,
c_datetime2 TIMESTAMP,
c_datetimeoffset TIMESTAMPTZ,
c_xml varchar,
PRIMARY KEY (id)
) from mssql_source table 'dbo.sqlserver_all_data_types';

Expand Down Expand Up @@ -306,7 +318,12 @@ CREATE TABLE upper_table (
"ID" INT,
"Name" VARCHAR,
PRIMARY KEY ("ID")
) from upper_mssql_source table 'UpperSchema.UpperTable';
)
INCLUDE TIMESTAMP AS commit_ts
INCLUDE DATABASE_NAME as database_name
INCLUDE SCHEMA_NAME as schema_name
INCLUDE TABLE_NAME as table_name
from upper_mssql_source table 'UpperSchema.UpperTable';

statement ok
create materialized view shared_orders_cnt as select count(*) as cnt from shared_orders;
Expand Down Expand Up @@ -349,11 +366,46 @@ SELECT * from shared_single_type order by id;
3 23:59:59.999

query TTTTTTT
SELECT * from shared_sqlserver_all_data_types order by id;
SELECT id, c_bit, c_tinyint, c_smallint, c_int, c_bigint from shared_sqlserver_all_data_types order by id;
----
1 f 0 0 0 0
2 t 255 -32768 -2147483648 -9223372036854775808
3 t 127 32767 2147483647 9223372036854775807

query TTTT
SELECT id, c_decimal, c_real, c_float from shared_sqlserver_all_data_types order by id;
----
1 0 0 0
2 -10 -10000 -10000
3 -10 10000 10000

query TTTTTTT
SELECT id, c_varchar, c_nvarchar, c_ntext, c_binary, c_varbinary, c_uniqueidentifier from shared_sqlserver_all_data_types order by id;
----
1 (empty) 中 中 \xff000000 NULL NULL
2 aa 🌹 🌹 NULL \xff 6F9619FF-8B86-D011-B42D-00C04FC964FF
3 zzzz 🌹👍 🌹👍 \xffffffff \xffffffff 6F9619FF-8B86-D011-B42D-00C04FC964FF

query TTT
SELECT id, char_length(c_char) from shared_sqlserver_all_data_types order by id;
----
1 4
2 4
3 4

query TTTTTTT
SELECT id, c_date, c_time, c_datetime2, c_datetimeoffset from shared_sqlserver_all_data_types order by id;
----
1 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
2 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
3 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
1 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
2 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
3 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00

query TTTTTTT
SELECT id, c_xml from shared_sqlserver_all_data_types order by id;
----
1 <Person><Name>John Doe</Name><Age>30</Age></Person>
2 <Person><Name>Jane Doe</Name><Age>28</Age></Person>
3 <Name>Jane Doe</Name>

# ------------ kill cluster ------------
# system ok
Expand Down Expand Up @@ -410,20 +462,80 @@ SELECT * from shared_single_type order by id;
13 23:59:59.999

query TTTTTTT
SELECT * from shared_sqlserver_all_data_types order by id;
SELECT id, c_bit, c_tinyint, c_smallint, c_int, c_bigint from shared_sqlserver_all_data_types order by id;
----
1 f 0 0 0 0
2 t 255 -32768 -2147483648 -9223372036854775808
3 t 127 32767 2147483647 9223372036854775807
11 f 0 0 0 0
12 t 255 -32768 -2147483648 -9223372036854775808
13 t 127 32767 2147483647 9223372036854775807

query TTTT
SELECT id, c_decimal, c_real, c_float from shared_sqlserver_all_data_types order by id;
----
1 0 0 0
2 -10 -10000 -10000
3 -10 10000 10000
11 0 0 0
12 -10 -10000 -10000
13 -10 10000 10000

query TTTTTTT
SELECT id, c_varchar, c_nvarchar, c_ntext, c_binary, c_varbinary, c_uniqueidentifier from shared_sqlserver_all_data_types order by id;
----
1 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
2 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
3 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
11 f 0 0 0 0 0 0 0 (empty) NULL 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
12 t 255 -32768 -2147483648 -9223372036854775808 -10 -10000 -10000 aa \xff 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
13 t 127 32767 2147483647 9223372036854775807 -10 10000 10000 zzzz \xffffffff 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00

query TT
SELECT * from upper_table order by "ID";
1 (empty) 中 中 \xff000000 NULL NULL
2 aa 🌹 🌹 NULL \xff 6F9619FF-8B86-D011-B42D-00C04FC964FF
3 zzzz 🌹👍 🌹👍 \xffffffff \xffffffff 6F9619FF-8B86-D011-B42D-00C04FC964FF
11 (empty) 中 中 \xff000000 NULL NULL
12 aa 🌹 🌹 NULL \xff 6F9619FF-8B86-D011-B42D-00C04FC964FF
13 zzzz 🌹👍 🌹👍 \xffffffff \xffffffff 6F9619FF-8B86-D011-B42D-00C04FC964FF

query TTT
SELECT id, char_length(c_char) from shared_sqlserver_all_data_types order by id;
----
1 4
2 4
3 4
11 4
12 4
13 4

query TTTTTTT
SELECT id, c_date, c_time, c_datetime2, c_datetimeoffset from shared_sqlserver_all_data_types order by id;
----
1 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
2 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
3 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00
11 2001-01-01 00:00:00 2001-01-01 00:00:00 2001-01-01 00:00:00+00:00
12 1990-01-01 13:59:59.123 2000-01-01 11:00:00.123 1990-01-01 00:00:01.123+00:00
13 2999-12-31 23:59:59.999 2099-12-31 23:59:59.999 2999-12-31 23:59:59.999+00:00

query TTTTTTT
SELECT id, c_xml from shared_sqlserver_all_data_types order by id;
----
1 <Person><Name>John Doe</Name><Age>30</Age></Person>
2 <Person><Name>Jane Doe</Name><Age>28</Age></Person>
3 <Name>Jane Doe</Name>
11 <Person><Name>John Doe</Name><Age>30</Age></Person>
12 <Person><Name>Jane Doe</Name><Age>28</Age></Person>
13 <Name>Jane Doe</Name>

query TTTTTTTT
SELECT "ID", "Name", database_name, schema_name, table_name from upper_table order by "ID";
----
1 Alice UpperDB UpperSchema UpperTable
11 Alice UpperDB UpperSchema UpperTable

query TTTTTTTT
SELECT commit_ts from upper_table where "ID" = 1;
----
1970-01-01 00:00:00+00:00

query TTTTTTTT
SELECT commit_ts > '2024-08-16 00:00:00.000+00:00' from upper_table where "ID" = 11;
----
1 Alice
11 Alice
t

# ------------ drop stage ------------
statement ok
Expand Down
7 changes: 3 additions & 4 deletions e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_insert.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,8 @@ VALUES

INSERT INTO single_type VALUES (13, '23:59:59.999')

INSERT INTO sqlserver_all_data_types VALUES (11, 'False', 0, 0, 0, 0, 0, 0, 0, '', '', N'中', N'中', 0xff, NULL, NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00', '<Person><Name>John Doe</Name><Age>30</Age></Person>');

INSERT INTO sqlserver_all_data_types VALUES (11, 'False', 0, 0, 0, 0, 0, 0, 0, '', NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00');
INSERT INTO sqlserver_all_data_types VALUES (12, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 'aa', N'🌹', N'🌹', NULL, 0xff, '6f9619ff-8b86-d011-b42d-00c04fc964ff', '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123', '<Person> <Name>Jane Doe</Name> <Age>28</Age> </Person>');

INSERT INTO sqlserver_all_data_types VALUES (12, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 0xff, '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123');

INSERT INTO sqlserver_all_data_types VALUES (13, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 0xffffffff, '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999')
INSERT INTO sqlserver_all_data_types VALUES (13, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 'zzzz', N'🌹👍', N'🌹👍', 0xffffffff, 0xffffffff, '6F9619FF-8B86-D011-B42D-00C04FC964FF', '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999', '<Name>Jane Doe</Name>')
14 changes: 10 additions & 4 deletions e2e_test/source/cdc_inline/sql_server_cdc/sql_server_cdc_prepare.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,24 +52,30 @@ CREATE TABLE sqlserver_all_data_types (
c_decimal DECIMAL(28),
c_real real,
c_float float,
c_char char(4),
c_varchar varchar(4),
c_nvarchar nvarchar(4),
c_ntext ntext,
c_binary binary(4),
c_varbinary varbinary(4),
c_uniqueidentifier uniqueidentifier,
c_date date,
c_time time,
c_datetime2 datetime2,
c_datetimeoffset datetimeoffset
c_datetimeoffset datetimeoffset,
c_xml xml
);

EXEC sys.sp_cdc_enable_table
@source_schema = 'dbo',
@source_name = 'sqlserver_all_data_types',
@role_name = NULL;

INSERT INTO sqlserver_all_data_types VALUES (1, 'False', 0, 0, 0, 0, 0, 0, 0, '', NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00');
INSERT INTO sqlserver_all_data_types VALUES (1, 'False', 0, 0, 0, 0, 0, 0, 0, '', '', N'中', N'中', 0xff, NULL, NULL, '2001-01-01', '00:00:00', '2001-01-01 00:00:00', '2001-01-01 00:00:00', '<Person><Name>John Doe</Name><Age>30</Age></Person>');

INSERT INTO sqlserver_all_data_types VALUES (2, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 0xff, '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123');
INSERT INTO sqlserver_all_data_types VALUES (2, 'True', 255, -32768, -2147483648, -9223372036854775808, -10.0, -9999.999999, -10000.0, 'aa', 'aa', N'🌹', N'🌹', NULL, 0xff, '6f9619ff-8b86-d011-b42d-00c04fc964ff', '1990-01-01', '13:59:59.123', '2000-01-01 11:00:00.123', '1990-01-01 00:00:01.123', '<Person> <Name>Jane Doe</Name> <Age>28</Age> </Person>');

INSERT INTO sqlserver_all_data_types VALUES (3, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 0xffffffff, '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999')
INSERT INTO sqlserver_all_data_types VALUES (3, 'True', 127, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzz', 'zzzz', N'🌹👍', N'🌹👍', 0xffffffff, 0xffffffff, '6F9619FF-8B86-D011-B42D-00C04FC964FF', '2999-12-31', '23:59:59.999', '2099-12-31 23:59:59.999', '2999-12-31 23:59:59.999', '<Name>Jane Doe</Name>')

-- Table without enabling CDC
CREATE TABLE orders_without_cdc (
Expand Down
9 changes: 8 additions & 1 deletion ...ctor-service/src/main/java/com/risingwave/connector/source/common/SqlServerValidator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,7 @@ private boolean isDataTypeCompatible(String ssDataType, Data.DataType.TypeName t
return Data.DataType.TypeName.INT16_VALUE <= val
&& val <= Data.DataType.TypeName.INT64_VALUE;
case "integer":
case "int":
return Data.DataType.TypeName.INT32_VALUE <= val
&& val <= Data.DataType.TypeName.INT64_VALUE;
case "bigint":
Expand All @@ -305,9 +306,15 @@ private boolean isDataTypeCompatible(String ssDataType, Data.DataType.TypeName t
case "decimal":
case "numeric":
return val == Data.DataType.TypeName.DECIMAL_VALUE;
case "char":
case "nchar":
case "varchar":
case "character varying":
case "nvarchar":
case "text":
case "ntext":
case "uniqueidentifier":
return val == Data.DataType.TypeName.VARCHAR_VALUE;
case "binary":
case "varbinary":
return val == Data.DataType.TypeName.BYTEA_VALUE;
case "date":
Expand Down
Loading
Loading