Merge branch 'main' into feat/migration-etcd-2-sql

Cargo.toml

@@ -116,7 +116,7 @@ aws-types = "1"
 etcd-client = { package = "madsim-etcd-client", version = "0.4" }
 futures-async-stream = "0.2.9"
 hytra = "0.1"
-rdkafka = { package = "madsim-rdkafka", version = "0.3.1", features = [
+rdkafka = { package = "madsim-rdkafka", version = "0.3.4", features = [
   "cmake-build",
 ] }
 hashbrown = { version = "0.14", features = ["ahash", "inline-more", "nightly"] }

ci/scripts/notify.py

@@ -11,44 +11,52 @@
 MAIN_CRON_TEST_MAP = {
     "test-notify": ["noelkwan", "noelkwan"],
     "test-notify-2": ["noelkwan", "noelkwan"],
-    "backfill-tests": ["noelkwan"],
-    "backwards-compat-tests": ["noelkwan"],
-    "sqlsmith-differential-tests": ["noelkwan"],
-    "fuzz-test": ["noelkwan"],
-    "run-micro-benchmarks": ["noelkwan"],
-    "upload-micro-benchmarks": ["noelkwan"],
-    "e2e-standalone-binary-tests": ["noelkwan"],
-    "e2e-single-node-binary-tests": ["pin", "peng", "noelkwan"],
-    "unit-test": ["zhi"],
-    "unit-test-deterministic": ["zhi"],
-    "e2e-test-release": ["zhi"],
-    "e2e-test-release-parallel": ["zhi"],
-    "e2e-test-release-parallel-memory": ["zhi"],
-    "integration-test-deterministic-scale": ["ziqi"],
-    "integration-test-deterministic-recovery": ["ziqi"],
-    "integration-test-deterministic-others": ["ziqi"],
-    "e2e-test-deterministic": ["runji"],
-    "recovery-test-deterministic": ["runji"],
-    "background-ddl-recovery-test-deterministic": ["runji"],
+    "docslt": ["tianxiao"],
+    "e2e-test-release": ["zhi", "Eric"],
+    "e2e-meta-backup-test-release": ["zhi", "Eric"],
+    "e2e-test-release-parallel": ["zhi", "Eric"],
+    "e2e-test-release-parallel-memory": ["zhi", "Eric"],
     "e2e-test-release-source": ["bohan", "siyuan"],
     "e2e-test-release-sink": ["bohan", "siyuan"],
-    "e2e-iceberg-sink-test": ["renjie"],
-    "e2e-iceberg-sink-v2-test": ["renjie"],
+    "fuzz-test": ["noelkwan"],
+    "unit-test": ["zhi", "Eric"],
+    "unit-test-deterministic": ["zhi", "Eric"],
+    "integration-test-deterministic-scale": ["ziqi", "Eric"],
+    "integration-test-deterministic-recovery": ["ziqi", "Eric"],
+    "integration-test-deterministic-backfill": ["ziqi", "Eric"],
+    "integration-test-deterministic-storage": ["ziqi", "Eric"],
+    "integration-test-deterministic-sink": ["ziqi", "Eric"],
+    "e2e-test-deterministic": ["runji", "noelkwan"],
+    "recovery-test-deterministic": ["runji", "noelkwan"],
+    "background-ddl-arrangement-backfill-recovery-test-deterministic": ["runji", "noelkwan"],
+    "background-ddl-recovery-test-deterministic": ["runji", "noelkwan"],
+    "e2e-iceberg-sink-test": ["zilin"],
+    "e2e-iceberg-sink-v2-test": ["zilin"],
     "e2e-java-binding-tests": ["yiming"],
-    "e2e-clickhouse-sink-tests": ["bohan","xinhao"],
     "s3-source-check-aws": ["bohan"],
     "s3-source-check-aws-json-parser": ["bohan"],
     "s3-source-check-aws-csv-parser": ["bohan"],
     "s3-v2-source-check-aws-json-parser": ["bohan"],
+    "s3-v2-source-batch-read-check-aws-json-parser": ["bohan"],
     "s3-v2-source-check-aws-csv-parser": ["bohan"],
-    "e2e-pulsar-sink-tests": ["renjie"],
     "s3-source-test-for-opendal-fs-engine-csv-parser": ["congyi", "kexiang"],
     "s3-source-test-for-opendal-fs-engine": ["congyi", "kexiang"],
-    "s3-source-tests": ["congyi", "kexiang"],
-    "pulsar-source-tests": ["renjie"],
-    "connector-node-integration-test": ["siyuan"],
+    "pulsar-source-tests": ["bohan"],
+    "run-micro-benchmarks": ["noelkwan"],
+    "upload-micro-benchmarks": ["noelkwan"],
+    "backwards-compat-tests": ["noelkwan"],
+    "sqlsmith-differential-tests": ["noelkwan"],
+    "backfill-tests": ["noelkwan"],
+    "e2e-standalone-binary-tests": ["noelkwan"],
+    "e2e-single-node-binary-tests": ["pin", "peng", "noelkwan"],
     "e2e-test-opendal-parallel": ["congyi"],
     "e2e-deltalake-sink-rust-tests": ["xinhao"],
+    "e2e-redis-sink-tests": ["xinhao"],
+    "e2e-starrocks-sink-tests": ["xinhao"],
+    "e2e-cassandra-sink-tests": ["xinhao"],
+    "e2e-clickhouse-sink-tests": ["bohan","xinhao"],
+    "e2e-pulsar-sink-tests": ["bohan"],
+    "connector-node-integration-test": ["siyuan"],
 }
 
 INTEGRATION_TEST_MAP = {
@@ -63,9 +71,12 @@
     "schema-registry-json": ["bohan"],
     "mysql-cdc-json": ["siyuan"],
     "postgres-cdc-json": ["siyuan"],
+    "mongodb-cdc-json": ["siyuan"],
     "mysql-sink-json": ["siyuan"],
     "postgres-sink-json": ["siyuan"],
-    "iceberg-cdc-json": ["renjie"],
+    "iceberg-cdc-json": ["zilin"],
+    "iceberg-sink-none": ["zilin"],
+    'iceberg-source-none': ["zilin"],
     "twitter-json": ["bohan"],
     "twitter-protobuf": ["bohan"],
     "twitter-pulsar-json": ["bohan"],
@@ -77,19 +88,21 @@
     "kinesis-s3-source-json": ["bohan"],
     "clickhouse-sink-json": ["xinhao"],
     "cockroach-sink-json": ["bohan"],
-    "kafka-cdc-sink-json": ["renjie"],
+    "kafka-cdc-sink-json": ["bohan"],
     "cassandra-and-scylladb-sink-json": ["xinhao"],
     "elasticsearch-sink-json": ["xinhao"],
     "redis-sink-json": ["xinhao"],
     "big-query-sink-json": ["xinhao"],
-    "vector-json": ["tao"],
+    "vector-json": ["wutao"],
+    "nats-json": ["wutao"],
+    "nats-protobuf": ["wutao"],
+    "mqtt-json": ["bohan"],
     "doris-sink-json": ["xinhao"],
     "starrocks-sink-json": ["xinhao"],
     "deltalake-sink-json": ["xinhao"],
     "pinot-sink-json": ["yiming"],
-    "client-library-none": ["tao"],
-    "nats-json": ["tao"],
-    "nats-protobuf": ["tao"],
+    "presto-trino-json": ["wutao"],
+    "client-library-none": ["wutao"],
 }
 
 def get_failed_tests(get_test_status, test_map):

docker/Dockerfile

@@ -3,10 +3,9 @@ FROM ubuntu:22.04 AS base
 ENV LANG en_US.utf8
 
 RUN apt-get update \
-  && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk
+  && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk software-properties-common
 
 # Install Python 3.12
-RUN apt-get -y install software-properties-common
 RUN add-apt-repository ppa:deadsnakes/ppa -y && \
     apt-get update -yy && \
     DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy