Merge remote-tracking branch 'origin/main' into siyuan/cdc-binlog-ret…

…ention

ci/scripts/pr-fuzz-test.sh

@@ -5,16 +5,6 @@ set -euo pipefail
 
 source ci/scripts/common.sh
 
-set +e
-# Set features, depending on our workflow
-# If sqlsmith files are modified, we run tests with sqlsmith enabled.
-MATCHES="ci/scripts/cron-fuzz-test.sh\
-\|ci/scripts/pr-fuzz-test.sh\
-\|ci/scripts/run-fuzz-test.sh\
-\|src/tests/sqlsmith"
-NOT_MATCHES="\.md"
-CHANGED=$(git diff --name-only origin/main | grep -v "$NOT_MATCHES" | grep "$MATCHES")
-set -e
 
 # NOTE(kwannoel): Disabled because there's some breakage after #12485,
 # see https://github.com/risingwavelabs/risingwave/issues/12577.
@@ -23,19 +13,7 @@ set -e
 export RUN_SQLSMITH_FRONTEND=0
 export RUN_SQLSMITH=1
 export SQLSMITH_COUNT=100
-
-# Run e2e tests if changes to sqlsmith source files detected.
-if [[ -n "$CHANGED" ]]; then
-    echo "--- Checking whether to run all sqlsmith tests"
-    echo "origin/main SHA: $(git rev-parse origin/main)"
-    echo "Changes to Sqlsmith source files detected:"
-    echo "$CHANGED"
-    export RUN_SQLSMITH=1
-    export SQLSMITH_COUNT=100
-    export TEST_NUM=32
-    echo "Enabled Sqlsmith tests."
-else
-    export RUN_SQLSMITH=0
-fi
+export TEST_NUM=32
+echo "Enabled Sqlsmith tests."
 
 source ci/scripts/run-fuzz-test.sh

ci/workflows/pull-request.yml

@@ -19,6 +19,7 @@ steps:
   - label: "build"
     command: "ci/scripts/build.sh -p ci-dev"
     key: "build"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-build")
     plugins:
       - docker-compose#v4.9.0:
           run: rw-build-env
@@ -30,6 +31,7 @@ steps:
   - label: "build other components"
     command: "ci/scripts/build-other.sh"
     key: "build-other"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-build-other")
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
@@ -46,6 +48,7 @@ steps:
   - label: "build (deterministic simulation)"
     command: "ci/scripts/build-simulation.sh"
     key: "build-simulation"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-build-simulation")
     plugins:
       - docker-compose#v4.9.0:
           run: rw-build-env
@@ -57,6 +60,7 @@ steps:
   - label: "docslt"
     command: "ci/scripts/docslt.sh"
     key: "docslt"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-docslt")
     plugins:
       - docker-compose#v4.9.0:
           run: rw-build-env
@@ -67,6 +71,7 @@ steps:
 
   - label: "end-to-end test"
     command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-e2e-test")
     depends_on:
       - "build"
       - "build-other"
@@ -82,6 +87,7 @@ steps:
 
   - label: "end-to-end test (parallel)"
     command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-e2e-parallel-tests")
     depends_on:
       - "build"
       - "docslt"
@@ -124,6 +130,7 @@ steps:
 
   - label: "end-to-end source test"
     command: "ci/scripts/e2e-source-test.sh -p ci-dev"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-e2e-source-tests")
     depends_on:
       - "build"
       - "build-other"
@@ -138,6 +145,7 @@ steps:
 
   - label: "end-to-end sink test"
     command: "ci/scripts/e2e-sink-test.sh -p ci-dev"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-e2e-sink-tests")
     depends_on:
       - "build"
       - "build-other"
@@ -248,6 +256,7 @@ steps:
 
   - label: "regress test"
     command: "ci/scripts/regress-test.sh -p ci-dev"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-regress-test")
     depends_on: "build"
     plugins:
       - docker-compose#v4.9.0:
@@ -263,6 +272,7 @@ steps:
   # This ensures our `main-cron` workflow will be stable.
   - label: "unit test"
     command: "ci/scripts/pr-unit-test.sh"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-unit-test")
     plugins:
       - ./ci/plugins/swapfile
       - seek-oss/aws-sm#v2.3.1:
@@ -278,6 +288,7 @@ steps:
 
   - label: "check"
     command: "ci/scripts/check.sh"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-check")
     plugins:
       - gencer/cache#v2.4.10:
           id: cache
@@ -299,6 +310,7 @@ steps:
 
   - label: "unit test (deterministic simulation)"
     command: "ci/scripts/deterministic-unit-test.sh"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation")
     plugins:
       - docker-compose#v4.9.0:
           run: rw-build-env
@@ -310,6 +322,7 @@ steps:
 
   - label: "integration test (deterministic simulation)"
     command: "TEST_NUM=5 ci/scripts/deterministic-it-test.sh"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation")
     depends_on: "build-simulation"
     plugins:
       - docker-compose#v4.9.0:
@@ -321,6 +334,7 @@ steps:
 
   - label: "end-to-end test (deterministic simulation)"
     command: "TEST_NUM=16 ci/scripts/deterministic-e2e-test.sh"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation")
     depends_on: "build-simulation"
     plugins:
       - seek-oss/aws-sm#v2.3.1:
@@ -339,6 +353,7 @@ steps:
 
   - label: "recovery test (deterministic simulation)"
     command: "TEST_NUM=8 KILL_RATE=0.5 ci/scripts/deterministic-recovery-test.sh"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation")
     depends_on: "build-simulation"
     plugins:
       # - seek-oss/aws-sm#v2.3.1:
@@ -358,6 +373,7 @@ steps:
 
   - label: "misc check"
     command: "ci/scripts/misc-check.sh"
+    if: (!build.pull_request.labels includes "ci/skip-ci" || build.pull_request.labels includes "ci/run-misc-check")
     plugins:
       - docker-compose#v4.9.0:
           run: rw-build-env
@@ -492,8 +508,10 @@ steps:
     timeout_in_minutes: 30
     retry: *auto-retry
 
+  # FIXME(kwannoel): Let the github PR labeller label it, if sqlsmith source files has changes.
   - label: "fuzz test"
     command: "ci/scripts/pr-fuzz-test.sh -p ci-dev"
+    if: build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests"
     depends_on:
       - "build"
       - "build-simulation"
@@ -506,3 +524,9 @@ steps:
       - ./ci/plugins/upload-failure-logs
     timeout_in_minutes: 15
     retry: *auto-retry
+
+  - label: "enable ci/skip-ci only in draft PRs"
+    if: build.pull_request.labels includes "ci/skip-ci" && !build.pull_request.draft
+    commands:
+      - echo "ci/skip-ci is only usable for draft Pull Requests"
+      - exit 1

src/connector/Cargo.toml

@@ -83,6 +83,7 @@ paste = "1"
 prometheus = { version = "0.13", features = ["process"] }
 prost = { version = "0.12", features = ["no-recursion-limit"] }
 prost-reflect = "0.12"
+prost-types = "0.12"
 protobuf-native = "0.2.1"
 pulsar = { version = "6.0", default-features = false, features = [
     "tokio-runtime",
@@ -138,7 +139,6 @@ workspace-hack = { path = "../workspace-hack" }
 
 [dev-dependencies]
 criterion = { workspace = true, features = ["async_tokio", "async"] }
-prost-types = "0.12"
 rand = "0.8"
 tempfile = "3"
 tracing-test = "0.2"

src/connector/src/lib.rs

@@ -32,6 +32,7 @@
 #![feature(impl_trait_in_assoc_type)]
 #![feature(iter_from_generator)]
 #![feature(if_let_guard)]
+#![feature(iterator_try_collect)]
 
 use std::time::Duration;