fix(integration_tests): use timestamptz for MySQL, TiDB, Doris and …

…StarRocks (#13641)
risingwavelabs · Nov 29, 2023 · 83e9586 · 83e9586
1 parent 56a7dc5
commit 83e9586
Show file tree

Hide file tree

Showing 19 changed files with 45 additions and 41 deletions.
diff --git a/integration_tests/doris-sink/README.md b/integration_tests/doris-sink/README.md
@@ -33,7 +33,7 @@ use demo;
 CREATE table demo_bhv_table(
     user_id int,
     target_id text,
-    event_timestamp datetime
+    event_timestamp_local datetime
 ) UNIQUE KEY(`user_id`)
 DISTRIBUTED BY HASH(`user_id`) BUCKETS 1
 PROPERTIES (

diff --git a/integration_tests/doris-sink/append-only-sql/create_mv.sql b/integration_tests/doris-sink/append-only-sql/create_mv.sql
@@ -2,6 +2,6 @@ CREATE MATERIALIZED VIEW bhv_mv AS
 SELECT
     user_id,
     target_id,
-    event_timestamp
+    event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local
 FROM
-    user_behaviors;
+    user_behaviors;
diff --git a/integration_tests/doris-sink/append-only-sql/create_source.sql b/integration_tests/doris-sink/append-only-sql/create_source.sql
@@ -2,7 +2,7 @@ CREATE table user_behaviors (
     user_id int,
     target_id VARCHAR,
     target_type VARCHAR,
-    event_timestamp TIMESTAMP,
+    event_timestamp TIMESTAMPTZ,
     behavior_type VARCHAR,
     parent_target_type VARCHAR,
     parent_target_id VARCHAR,
@@ -15,4 +15,4 @@ CREATE table user_behaviors (
     fields.user_name.kind = 'random',
     fields.user_name.length = '10',
     datagen.rows.per.second = '10'
-) FORMAT PLAIN ENCODE JSON;
+) FORMAT PLAIN ENCODE JSON;
diff --git a/integration_tests/doris-sink/upsert/create_mv.sql b/integration_tests/doris-sink/upsert/create_mv.sql
@@ -2,6 +2,6 @@ CREATE MATERIALIZED VIEW bhv_mv AS
 SELECT
     user_id,
     target_id,
-    event_timestamp
+    event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local
 FROM
-    user_behaviors;
+    user_behaviors;
diff --git a/integration_tests/doris-sink/upsert/create_table.sql b/integration_tests/doris-sink/upsert/create_table.sql
@@ -2,9 +2,9 @@ CREATE table user_behaviors (
     user_id int,
     target_id VARCHAR,
     target_type VARCHAR,
-    event_timestamp TIMESTAMP,
+    event_timestamp TIMESTAMPTZ,
     behavior_type VARCHAR,
     parent_target_type VARCHAR,
     parent_target_id VARCHAR,
     PRIMARY KEY(user_id)
-);
+);
diff --git a/integration_tests/doris-sink/upsert/insert_update_delete.sql b/integration_tests/doris-sink/upsert/insert_update_delete.sql
@@ -1,8 +1,8 @@
-INSERT INTO user_behaviors VALUES(1,'1','1','2020-01-01 01:01:01','1','1','1'),
-(2,'2','2','2020-01-01 01:01:02','2','2','2'),
-(3,'3','3','2020-01-01 01:01:03','3','3','3'),
-(4,'4','4','2020-01-01 01:01:04','4','4','4');
+INSERT INTO user_behaviors VALUES(1,'1','1','2020-01-01T01:01:01Z','1','1','1'),
+(2,'2','2','2020-01-01T01:01:02Z','2','2','2'),
+(3,'3','3','2020-01-01T01:01:03Z','3','3','3'),
+(4,'4','4','2020-01-01T01:01:04Z','4','4','4');
 
 DELETE FROM user_behaviors WHERE user_id = 2;
 
-UPDATE user_behaviors SET target_id = 30 WHERE user_id = 3;
+UPDATE user_behaviors SET target_id = 30 WHERE user_id = 3;
diff --git a/integration_tests/mysql-sink/create_mv.sql b/integration_tests/mysql-sink/create_mv.sql
@@ -31,7 +31,8 @@ CREATE TABLE rw_typed_data (
     boolean_column BOOLEAN,
     date_column DATE,
     time_column TIME,
-    timestamp_column TIMESTAMPTZ,
+    timestamp_column TIMESTAMP,
+    timestamptz_column TIMESTAMPTZ,
     jsonb_column JSONB,
     bytea_column BYTEA
 ) WITH (

diff --git a/integration_tests/mysql-sink/create_source.sql b/integration_tests/mysql-sink/create_source.sql
@@ -27,6 +27,7 @@ CREATE TABLE data_types (
     date_column DATE,
     time_column TIME,
     timestamp_column TIMESTAMP,
+    timestamptz_column TIMESTAMPTZ,
     jsonb_column JSONB,
     bytea_column BYTEA
 );
@@ -41,10 +42,10 @@ FROM
         primary_key = 'id'
     );
 
-INSERT INTO data_types (id, varchar_column, text_column, integer_column, smallint_column, bigint_column, decimal_column, real_column, double_column, boolean_column, date_column, time_column, timestamp_column, jsonb_column, bytea_column)
+INSERT INTO data_types (id, varchar_column, text_column, integer_column, smallint_column, bigint_column, decimal_column, real_column, double_column, boolean_column, date_column, time_column, timestamp_column, timestamptz_column, jsonb_column, bytea_column)
 VALUES
-    (1, 'Varchar value 1', 'Text value 1', 123, 456, 789, 12.34, 56.78, 90.12, TRUE, '2023-05-22', '12:34:56', '2023-05-22 12:34:56', '{"key": "value"}', E'\\xDEADBEEF'),
-    (2, 'Varchar value 2', 'Text value 2', 234, 567, 890, 23.45, 67.89, 01.23, FALSE, '2023-05-23', '23:45:01', '2023-05-23 23:45:01', '{"key": "value2"}', E'\\xFEEDBEEF'),
-    (3, 'Varchar value 3', 'Text value 3', 345, 678, 901, 34.56, 78.90, 12.34, TRUE, '2023-05-24', '12:34:56', '2023-05-24 12:34:56', '{"key": "value3"}', E'\\xCAFEBABE'),
-    (4, 'Varchar value 4', 'Text value 4', 456, 789, 012, 45.67, 89.01, 23.45, FALSE, '2023-05-25', '23:45:01', '2023-05-25 23:45:01', '{"key": "value4"}', E'\\xBABEC0DE'),
-    (5, 'Varchar value 5', 'Text value 5', 567, 890, 123, 56.78, 90.12, 34.56, TRUE, '2023-05-26', '12:34:56', '2023-05-26 12:34:56', '{"key": "value5"}', E'\\xDEADBABE');
+    (1, 'Varchar value 1', 'Text value 1', 123, 456, 789, 12.34, 56.78, 90.12, TRUE, '2023-05-22', '12:34:56', '2023-05-22 12:34:56', '2023-05-22T12:34:56Z', '{"key": "value"}', E'\\xDEADBEEF'),
+    (2, 'Varchar value 2', 'Text value 2', 234, 567, 890, 23.45, 67.89, 01.23, FALSE, '2023-05-23', '23:45:01', '2023-05-23 23:45:01', '2023-05-23T23:45:01Z', '{"key": "value2"}', E'\\xFEEDBEEF'),
+    (3, 'Varchar value 3', 'Text value 3', 345, 678, 901, 34.56, 78.90, 12.34, TRUE, '2023-05-24', '12:34:56', '2023-05-24 12:34:56', '2023-05-24T12:34:56Z', '{"key": "value3"}', E'\\xCAFEBABE'),
+    (4, 'Varchar value 4', 'Text value 4', 456, 789, 012, 45.67, 89.01, 23.45, FALSE, '2023-05-25', '23:45:01', '2023-05-25 23:45:01', '2023-05-25T23:45:01Z', '{"key": "value4"}', E'\\xBABEC0DE'),
+    (5, 'Varchar value 5', 'Text value 5', 567, 890, 123, 56.78, 90.12, 34.56, TRUE, '2023-05-26', '12:34:56', '2023-05-26 12:34:56', '2023-05-26T12:34:56Z', '{"key": "value5"}', E'\\xDEADBABE');
diff --git a/integration_tests/mysql-sink/mysql_prepare.sql b/integration_tests/mysql-sink/mysql_prepare.sql
@@ -18,7 +18,8 @@ CREATE TABLE data_types (
     boolean_column BOOLEAN,
     date_column DATE,
     time_column TIME,
-    timestamp_column TIMESTAMP,
+    timestamp_column DATETIME,
+    timestamptz_column TIMESTAMP,
     jsonb_column JSON,
     bytea_column BLOB
-);
+);
diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py
@@ -14,7 +14,7 @@ def run_sql_file(f: str, dir: str):
     # ON_ERROR_STOP=1 will let psql return error code when the query fails.
     # https://stackoverflow.com/questions/37072245/check-return-status-of-psql-command-in-unix-shell-scripting
     proc = subprocess.run(["psql", "-h", "localhost", "-p", "4566",
-                           "-d", "dev", "-U", "root", "-f", f, "-v", "ON_ERROR_STOP=1"], check=True,
+                           "-d", "dev", "-U", "root", "-f", f, "-v", "ON_ERROR_STOP=1", "-P", "pager=off"], check=True,
                           cwd=dir)
     if proc.returncode != 0:
         sys.exit(1)

diff --git a/integration_tests/starrocks-sink/README.md b/integration_tests/starrocks-sink/README.md
@@ -26,7 +26,7 @@ use demo;
 CREATE table demo_bhv_table(
     user_id int,
     target_id text,
-    event_timestamp datetime
+    event_timestamp_local datetime
 ) ENGINE=OLAP
 PRIMARY KEY(`user_id`)
 DISTRIBUTED BY HASH(`user_id`) properties("replication_num" = "1");

diff --git a/integration_tests/starrocks-sink/append-only-sql/create_mv.sql b/integration_tests/starrocks-sink/append-only-sql/create_mv.sql
@@ -2,6 +2,6 @@ CREATE MATERIALIZED VIEW bhv_mv AS
 SELECT
     user_id,
     target_id,
-    event_timestamp
+    event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local
 FROM
-    user_behaviors;
+    user_behaviors;
diff --git a/integration_tests/starrocks-sink/append-only-sql/create_source.sql b/integration_tests/starrocks-sink/append-only-sql/create_source.sql
@@ -2,7 +2,7 @@ CREATE table user_behaviors (
     user_id int,
     target_id VARCHAR,
     target_type VARCHAR,
-    event_timestamp TIMESTAMP,
+    event_timestamp TIMESTAMPTZ,
     behavior_type VARCHAR,
     parent_target_type VARCHAR,
     parent_target_id VARCHAR,
@@ -15,4 +15,4 @@ CREATE table user_behaviors (
     fields.user_name.kind = 'random',
     fields.user_name.length = '10',
     datagen.rows.per.second = '10'
-) FORMAT PLAIN ENCODE JSON;
+) FORMAT PLAIN ENCODE JSON;
diff --git a/integration_tests/starrocks-sink/upsert/create_mv.sql b/integration_tests/starrocks-sink/upsert/create_mv.sql
@@ -2,6 +2,6 @@ CREATE MATERIALIZED VIEW bhv_mv AS
 SELECT
     user_id,
     target_id,
-    event_timestamp
+    event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local
 FROM
-    user_behaviors;
+    user_behaviors;
diff --git a/integration_tests/starrocks-sink/upsert/create_table.sql b/integration_tests/starrocks-sink/upsert/create_table.sql
@@ -2,9 +2,9 @@ CREATE table user_behaviors (
     user_id int,
     target_id VARCHAR,
     target_type VARCHAR,
-    event_timestamp TIMESTAMP,
+    event_timestamp TIMESTAMPTZ,
     behavior_type VARCHAR,
     parent_target_type VARCHAR,
     parent_target_id VARCHAR,
     PRIMARY KEY(user_id)
-);
+);
diff --git a/integration_tests/starrocks-sink/upsert/insert_update_delete.sql b/integration_tests/starrocks-sink/upsert/insert_update_delete.sql
@@ -1,8 +1,8 @@
-INSERT INTO user_behaviors VALUES(1,'1','1','2020-01-01 01:01:01','1','1','1'),
-(2,'2','2','2020-01-01 01:01:02','2','2','2'),
-(3,'3','3','2020-01-01 01:01:03','3','3','3'),
-(4,'4','4','2020-01-01 01:01:04','4','4','4');
+INSERT INTO user_behaviors VALUES(1,'1','1','2020-01-01T01:01:01Z','1','1','1'),
+(2,'2','2','2020-01-01T01:01:02Z','2','2','2'),
+(3,'3','3','2020-01-01T01:01:03Z','3','3','3'),
+(4,'4','4','2020-01-01T01:01:04Z','4','4','4');
 
 DELETE FROM user_behaviors WHERE user_id = 2;
 
-UPDATE user_behaviors SET target_id = 30 WHERE user_id = 3;
+UPDATE user_behaviors SET target_id = 30 WHERE user_id = 3;
diff --git a/integration_tests/tidb-cdc-sink/create_mv.sql b/integration_tests/tidb-cdc-sink/create_mv.sql
@@ -4,7 +4,7 @@
 CREATE MATERIALIZED VIEW hot_hashtags AS WITH tags AS (
     SELECT
         unnest(regexp_matches(tweet.text, '#\w+', 'g')) AS hashtag,
-        tweet.created_at AS created_at
+        tweet.created_at AT TIME ZONE 'UTC' AS created_at
     FROM
         tweet JOIN user
     ON

diff --git a/integration_tests/tidb-cdc-sink/create_source.sql b/integration_tests/tidb-cdc-sink/create_source.sql
@@ -2,7 +2,7 @@ CREATE TABLE tweet (
   id varchar,
   text varchar,
   lang varchar,
-  created_at timestamp,
+  created_at timestamp, -- #13682: shall be timestamptz from TiDB timestamp, but canal is problematic
   author_id varchar,
   PRIMARY KEY (id)
 ) WITH (

diff --git a/integration_tests/tidb-cdc-sink/docker-compose.yml b/integration_tests/tidb-cdc-sink/docker-compose.yml
@@ -67,6 +67,7 @@ services:
       - --addr=0.0.0.0:8300
       - --pd=http://pd:2379
       - --advertise-addr=ticdc-capturer0:8300
+      - --tz=UTC
     depends_on:
       - pd
       - "tidb"
@@ -229,4 +230,4 @@ volumes:
   prometheus-0:
     external: false
   message_queue:
-    external: false
+    external: false