Skip to content

Commit

Permalink
resolve conflicts
Browse files Browse the repository at this point in the history
  • Loading branch information
chenzl25 committed Nov 26, 2024
2 parents 7a10d06 + a3c2648 commit 41726fb
Show file tree
Hide file tree
Showing 95 changed files with 1,256 additions and 703 deletions.
16 changes: 16 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,24 @@ updates:
- dependency-name: "io.debezium:*"
update-types:
["version-update:semver-minor", "version-update:semver-major"]
# Don't upgrade protobuf to 4.x now. See https://github.com/grpc/grpc-java/issues/11015
- dependency-name: "com.google.protobuf:*"
update-types:
["version-update:semver-major"]
# Let's do major version updates manually
- dependency-name: "*"
update-types:
["version-update:semver-major"]
groups:
# Group all dependenies together because Java libraries are quite stable
all:
patterns:
- "*"

# Don't touch risingwave-sink-deltalake-test. It's too complicated and it's only for testing
- package-ecosystem: maven
directory: /java/connector-node/risingwave-sink-deltalake-test/
schedule:
interval: "weekly"
ignore:
- dependency-name: "*"
8 changes: 4 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ parquet = { version = "53.2", features = ["async"] }
mysql_async = { version = "0.34", default-features = false, features = [
"default",
] }
thiserror-ext = "0.1.2"
thiserror-ext = { version = "0.2.1", features = ["backtrace"] }
tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" }
tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [
"profiling",
Expand Down
14 changes: 7 additions & 7 deletions ci/workflows/main-cron.yml
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ steps:
files: "*-junit.xml"
format: "junit"
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 11
timeout_in_minutes: 13
retry: *auto-retry

- label: "end-to-end test (parallel, in-memory) (release)"
Expand Down Expand Up @@ -347,7 +347,7 @@ steps:

- label: "end-to-end test (madsim)"
key: "e2e-test-deterministic"
command: "TEST_NUM=32 timeout 120m ci/scripts/deterministic-e2e-test.sh"
command: "TEST_NUM=32 timeout 130m ci/scripts/deterministic-e2e-test.sh"
if: |
!(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
|| build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation"
Expand All @@ -364,12 +364,12 @@ steps:
environment:
- GITHUB_TOKEN
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 120
timeout_in_minutes: 135
retry: *auto-retry

- label: "end-to-end test (madsim, random vnode count)"
key: "e2e-test-deterministic-random-vnode-count"
command: "TEST_NUM=32 RW_SIM_RANDOM_VNODE_COUNT=true timeout 120m ci/scripts/deterministic-e2e-test.sh"
command: "TEST_NUM=32 RW_SIM_RANDOM_VNODE_COUNT=true timeout 130m ci/scripts/deterministic-e2e-test.sh"
if: |
!(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
|| build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation"
Expand All @@ -386,12 +386,12 @@ steps:
environment:
- GITHUB_TOKEN
- ./ci/plugins/upload-failure-logs
timeout_in_minutes: 120
timeout_in_minutes: 135
retry: *auto-retry

- label: "recovery test (madsim)"
key: "recovery-test-deterministic"
command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 timeout 65m ci/scripts/deterministic-recovery-test.sh"
command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 timeout 70m ci/scripts/deterministic-recovery-test.sh"
if: |
!(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
|| build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation"
Expand All @@ -404,7 +404,7 @@ steps:
mount-buildkite-agent: true
# Only upload zipped files, otherwise the logs is too much.
- ./ci/plugins/upload-failure-logs-zipped
timeout_in_minutes: 70
timeout_in_minutes: 75
retry: *auto-retry

# Ddl statements will randomly run with background_ddl.
Expand Down
24 changes: 21 additions & 3 deletions e2e_test/batch/basic/dml_update.slt.part
Original file line number Diff line number Diff line change
Expand Up @@ -93,10 +93,28 @@ select * from t;
889 999

# Multiple assignments, to subquery with cast failure.
# TODO: this currently shows `cannot cast type "record" to "record"` because we wrap the subquery result
# into a struct, which is not quite clear.
statement error cannot cast type
statement error
update t set (v1, v2) = (select '888.88', 999);
----
db error: ERROR: Failed to run the query

Caused by these errors (recent errors listed first):
1: cannot cast type "record" to "struct<v1 integer, v2 integer>"
2: cannot cast to struct field "v1"
3: cannot cast type "character varying" to "integer" in Assign context


# Multiple assignments, to subquery (with column name) with cast failure.
statement error
update t set (v1, v2) = (select '888.88' s1, 999 s2);
----
db error: ERROR: Failed to run the query

Caused by these errors (recent errors listed first):
1: cannot cast type "struct<s1 character varying, s2 integer>" to "struct<v1 integer, v2 integer>"
2: cannot cast struct field "s1" to struct field "v1"
3: cannot cast type "character varying" to "integer" in Assign context


# Multiple assignments, to subquery with mismatched column count.
statement error number of columns does not match number of values
Expand Down
133 changes: 121 additions & 12 deletions e2e_test/python_client/main.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,128 @@
import psycopg
from decimal import Decimal
import math
import unittest
import datetime
import zoneinfo

def test_psycopg_extended_mode():
conn = psycopg.connect(host='localhost', port='4566', dbname='dev', user='root')
with conn.cursor() as cur:
cur.execute("select Array[1::bigint, 2::bigint, 3::bigint]", binary=True)
assert cur.fetchone() == ([1, 2, 3],)
PG_HOST = 'localhost'
PG_PORT = 4566
PG_DBNAME = 'dev'
PG_USER = 'root'

cur.execute("select Array['foo', null, 'bar']", binary=True)
assert cur.fetchone() == (['foo', None, 'bar'],)
class TestPsycopgExtendedMode(unittest.TestCase):
def test_psycopg_extended_mode(self):
with psycopg.connect(host=PG_HOST, port=PG_PORT, dbname=PG_DBNAME, user=PG_USER) as conn:
with conn.cursor() as cur:
# Boolean
cur.execute("select true, false, null::boolean", binary=True)
self.assertEqual(cur.fetchone(), (True, False, None))

cur.execute("select ROW('123 Main St', 'New York', '10001')", binary=True)
assert cur.fetchone() == (('123 Main St', 'New York', '10001'),)
# Integer types
cur.execute("select 1::smallint, 2::integer, 3::bigint", binary=True)
self.assertEqual(cur.fetchone(), (1, 2, 3))

cur.execute("select array[ROW('123 Main St', 'New York', '10001'), ROW('234 Main St', null, '10001')]", binary=True)
assert cur.fetchone() == ([('123 Main St', 'New York', '10001'), ('234 Main St', None, '10001')],)
# Decimal/Numeric types
cur.execute("select 1.23::decimal, 2.5::real, 3.45::double precision", binary=True)
self.assertEqual(cur.fetchone(), (Decimal('1.23'), 2.5, 3.45))

# String
cur.execute("select 'hello'::varchar, null::varchar", binary=True)
self.assertEqual(cur.fetchone(), ('hello', None))

# Date/Time types
cur.execute("select '2023-01-01'::date, '12:34:56'::time, '2023-01-01 12:34:56'::timestamp, '2023-01-01 12:34:56+00'::timestamptz", binary=True)
self.assertEqual(cur.fetchone(), (datetime.date(2023, 1, 1), datetime.time(12, 34, 56), datetime.datetime(2023, 1, 1, 12, 34, 56), datetime.datetime(2023, 1, 1, 20, 34, 56, tzinfo=zoneinfo.ZoneInfo(key='Asia/Shanghai'))))

# Interval
cur.execute("select '1 year 2 months 3 days 4 hours 5 minutes 6 seconds'::interval", binary=True)
self.assertEqual(cur.fetchone(), (datetime.timedelta(days=428, seconds=14706),))

# Byte array
cur.execute("select '\\xDEADBEEF'::bytea", binary=True)
self.assertEqual(cur.fetchone(), (b'\xDE\xAD\xBE\xEF',))

cur.execute("select '\\x'::bytea", binary=True)
self.assertEqual(cur.fetchone(), (b'',))

# Array
cur.execute("select ARRAY[true, false, null]::boolean[]", binary=True)
self.assertEqual(cur.fetchone(), ([True, False, None],))

cur.execute("select ARRAY[1, 2, 3]::smallint[]", binary=True)
self.assertEqual(cur.fetchone(), ([1, 2, 3],))

cur.execute("select ARRAY[1, 2, 3]::integer[]", binary=True)
self.assertEqual(cur.fetchone(), ([1, 2, 3],))

cur.execute("select ARRAY[1, 2, 3]::bigint[]", binary=True)
self.assertEqual(cur.fetchone(), ([1, 2, 3],))

cur.execute("select ARRAY[1.1, 2.2, 3.3]::decimal[]", binary=True)
self.assertEqual(cur.fetchone(), ([Decimal('1.1'), Decimal('2.2'), Decimal('3.3')],))

cur.execute("select ARRAY[1.1, 2.2, 3.3]::real[]", binary=True)
result = cur.fetchone()[0] # Fetch once and store the result
self.assertAlmostEqual(result[0], 1.1, places=6)
self.assertAlmostEqual(result[1], 2.2, places=6)
self.assertAlmostEqual(result[2], 3.3, places=6)

cur.execute("select ARRAY[1.1, 2.2, 3.3]::double precision[]", binary=True)
result = cur.fetchone()[0] # Fetch once and store the result
self.assertAlmostEqual(result[0], 1.1, places=6)
self.assertAlmostEqual(result[1], 2.2, places=6)
self.assertAlmostEqual(result[2], 3.3, places=6)

cur.execute("select ARRAY['foo', 'bar', null]::varchar[]", binary=True)
self.assertEqual(cur.fetchone(), (['foo', 'bar', None],))

cur.execute("select ARRAY['\\xDEADBEEF'::bytea, '\\x0102'::bytea]", binary=True)
self.assertEqual(cur.fetchone(), ([b'\xDE\xAD\xBE\xEF', b'\x01\x02'],))

cur.execute("select ARRAY['2023-01-01', '2023-01-02']::date[]", binary=True)
self.assertEqual(cur.fetchone(), ([datetime.date(2023, 1, 1), datetime.date(2023, 1, 2)],))

cur.execute("select ARRAY['12:34:56', '23:45:01']::time[]", binary=True)
self.assertEqual(cur.fetchone()[0], [datetime.time(12, 34, 56), datetime.time(23, 45, 1)])

cur.execute("select ARRAY['2023-01-01 12:34:56', '2023-01-02 23:45:01']::timestamp[]", binary=True)
self.assertEqual(cur.fetchone()[0], [datetime.datetime(2023, 1, 1, 12, 34, 56), datetime.datetime(2023, 1, 2, 23, 45, 1)])

cur.execute("select ARRAY['2023-01-01 12:34:56+00', '2023-01-02 23:45:01+00']::timestamptz[]", binary=True)
self.assertEqual(cur.fetchone()[0], [datetime.datetime(2023, 1, 1, 12, 34, 56, tzinfo=datetime.timezone.utc), datetime.datetime(2023, 1, 2, 23, 45, 1, tzinfo=datetime.timezone.utc)])

cur.execute("select ARRAY['{\"a\": 1}'::jsonb, '{\"b\": 2}'::jsonb]", binary=True)
self.assertEqual(cur.fetchone(), ([{'a': 1}, {'b': 2}],))

# Struct
cur.execute("select ROW('123 Main St'::varchar, 'New York'::varchar, 10001)", binary=True)
self.assertEqual(cur.fetchone(), (('123 Main St', 'New York', 10001),))

cur.execute("select array[ROW('123 Main St'::varchar, 'New York'::varchar, 10001), ROW('234 Main St'::varchar, null, 10002)]", binary=True)
self.assertEqual(cur.fetchone(), ([('123 Main St', 'New York', 10001), ('234 Main St', None, 10002)],))

# Numeric
cur.execute("select 'NaN'::numeric, 'NaN'::real, 'NaN'::double precision", binary=True)
result = cur.fetchone()
self.assertTrue(result[0].is_nan())
self.assertTrue(math.isnan(result[1]))
self.assertTrue(math.isnan(result[2]))

cur.execute("select 'Infinity'::numeric, 'Infinity'::real, 'Infinity'::double precision", binary=True)
self.assertEqual(cur.fetchone(), (float('inf'), float('inf'), float('inf')))

cur.execute("select '-Infinity'::numeric, '-Infinity'::real, '-Infinity'::double precision", binary=True)
self.assertEqual(cur.fetchone(), (float('-inf'), float('-inf'), float('-inf')))

# JSONB
cur.execute("select '{\"name\": \"John\", \"age\": 30, \"city\": null}'::jsonb", binary=True)
self.assertEqual(cur.fetchone(), ({'name': 'John', 'age': 30, 'city': None},))

cur.execute("select '{\"scores\": [85.5, 90, null], \"passed\": true}'::jsonb", binary=True)
self.assertEqual(cur.fetchone(), ({'scores': [85.5, 90, None], 'passed': True},))

cur.execute("select '[{\"id\": 1, \"value\": null}, {\"id\": 2, \"value\": \"test\"}]'::jsonb", binary=True)
self.assertEqual(cur.fetchone(), ([{'id': 1, 'value': None}, {'id': 2, 'value': 'test'}],))

if __name__ == '__main__':
test_psycopg_extended_mode()
unittest.main()
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.risingwave.proto.Catalog.Table;
import com.risingwave.proto.ClusterServiceGrpc.ClusterServiceBlockingStub;
import com.risingwave.proto.Common.HostAddress;
import com.risingwave.proto.Common.WorkerNode.Property;
import com.risingwave.proto.Common.WorkerType;
import com.risingwave.proto.DdlServiceGrpc.DdlServiceBlockingStub;
import com.risingwave.proto.DdlServiceOuterClass.GetTableRequest;
Expand All @@ -29,7 +30,6 @@
import com.risingwave.proto.Hummock.UnpinVersionBeforeRequest;
import com.risingwave.proto.HummockManagerServiceGrpc.HummockManagerServiceBlockingStub;
import com.risingwave.proto.Meta.AddWorkerNodeRequest;
import com.risingwave.proto.Meta.AddWorkerNodeRequest.Property;
import com.risingwave.proto.Meta.AddWorkerNodeResponse;
import com.risingwave.proto.Meta.HeartbeatRequest;
import io.grpc.Grpc;
Expand Down Expand Up @@ -100,7 +100,6 @@ public MetaClient(String metaAddr, ScheduledExecutorService scheduler) {
Property.newBuilder()
.setIsStreaming(false)
.setIsServing(false)
.setWorkerNodeParallelism(0)
.build())
.build();
AddWorkerNodeResponse resp = clusterStub.addWorkerNode(req);
Expand Down
2 changes: 1 addition & 1 deletion java/connector-node/assembly/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.4.2</version>
<version>3.7.1</version>
<configuration>
<encoding>UTF-8</encoding>
<finalName>${assembly.name}</finalName>
Expand Down
1 change: 0 additions & 1 deletion java/connector-node/risingwave-sink-deltalake/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<parquet.version>1.14.4</parquet.version>
<maven.javadoc.skip>true</maven.javadoc.skip>
</properties>

Expand Down
4 changes: 2 additions & 2 deletions java/connector-node/risingwave-sink-iceberg/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<name>risingwave-sink-iceberg</name>

<properties>
<iceberg.version>1.5.2</iceberg.version>
<iceberg.version>1.7.0</iceberg.version>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<maven.javadoc.skip>true</maven.javadoc.skip>
Expand Down Expand Up @@ -77,7 +77,7 @@
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-avro</artifactId>
<version>1.14.3</version>
<version>${parquet.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down
8 changes: 1 addition & 7 deletions java/connector-node/risingwave-source-cdc-test/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>6.0.0</version>
<version>6.2.1</version>
<scope>test</scope>
</dependency>
<dependency>
Expand All @@ -76,12 +76,6 @@
<groupId>org.testcontainers</groupId>
<artifactId>mongodb</artifactId>
</dependency>

<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-sync</artifactId>
<version>5.2.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
Expand Down
1 change: 1 addition & 0 deletions java/connector-node/risingwave-source-cdc/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-sync</artifactId>
<version>4.11.5</version>
</dependency>
</dependencies>
</project>
2 changes: 1 addition & 1 deletion java/java-binding/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<module.version>0.1.0-SNAPSHOT</module.version>
<jni.loader.version>1.0.0</jni.loader.version>
<jni.loader.version>1.2.0</jni.loader.version>
<java.binding.release>false</java.binding.release>
<maven.javadoc.skip>true</maven.javadoc.skip>
</properties>
Expand Down
Loading

0 comments on commit 41726fb

Please sign in to comment.